Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
refactor(control-api): clean up before deleting an mcp-server CR and fence context attach (R3-H7)
Diego Zacarias178985b4087a
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 0a6faf059..1fea030c8 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.668",+ "version": "0.1.669","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.668",+ "version": "0.1.669","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex b552ba4c6..afd6d4008 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.668",+ "version": "0.1.669","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/oauth/mcpServerOAuthTeardown.ts b/control-api/src/oauth/mcpServerOAuthTeardown.tsindex 11e45e728..e49ae9201 100644--- a/control-api/src/oauth/mcpServerOAuthTeardown.ts+++ b/control-api/src/oauth/mcpServerOAuthTeardown.ts@@ -20,9 +20,10 @@ import { deleteOAuthGrantsForServer } from './store.js'* - C: the old read→await(7592)→delete gap let a reinstall slip a new row in between.* Here there is no gap: the claim is one statement.*- * Runs AFTER the CR delete; never throws. Each stage is best-effort — a throw is- * caught, counted (metric, existing stage labels) and logged (names-only), and the- * uninstall keeps its 200. Changing that response policy is R3-H7, out of scope.+ * Runs while the CR still exists (see mcpServerUninstall.ts) and never throws: a+ * throw is caught, counted (metric, per stage) and logged (names-only), and reported+ * as `'failed'` so the caller can keep the CR and answer "repair required" — the+ * retry reuses the same uid.*/export type TeardownStageResult = 'done' | 'none' | 'failed'@@ -70,10 +71,7 @@ export async function teardownMcpServerOAuthState(} catch (err) {result.dynamicClient = 'failed'mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'dynamic_client' })- logger.error(- { serverName: name, namespace, err },- 'Dynamic client cleanup failed on uninstall (CR already deleted)'- )+ logger.error({ serverName: name, namespace, err }, 'Dynamic client cleanup failed on uninstall')}// Grants: purge this installation's rows (fenced by cr_uid + legacy). A reinstall's@@ -94,10 +92,7 @@ export async function teardownMcpServerOAuthState(} catch (err) {result.grants = 'failed'mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants' })- logger.error(- { serverName: name, namespace, err },- 'OAuth grants purge failed on uninstall (CR already deleted)'- )+ logger.error({ serverName: name, namespace, err }, 'OAuth grants purge failed on uninstall')}return resultdiff --git a/control-api/src/oauth/mcpServerUninstall.ts b/control-api/src/oauth/mcpServerUninstall.tsnew file mode 100644index 000000000..3fe438624--- /dev/null+++ b/control-api/src/oauth/mcpServerUninstall.ts@@ -0,0 +1,258 @@+import type { DbClient } from '../db.js'+import { extractK8sError } from '../http/k8sError.js'+import type { K8sGateway } from '../k8s.js'+import type { Logger } from '../observability/logger.js'+import { mcpServerUninstallTeardownFailuresTotal } from '../observability/metrics.js'+import { isNotFound, stripServerFromContexts } from '../services/contextAllowlist.js'+import { type SecretCleanupCapture, captureSecretForCleanup } from '../services/secretCleanup.js'+import type { DcrDeps } from './dcr.js'+import { teardownMcpServerOAuthState } from './mcpServerOAuthTeardown.js'+import { deleteOAuthGrantsForServer } from './store.js'++/**+ * Uninstall of one McpServer with every cleanup BEFORE the CR delete.+ *+ * Why this order: the CR's `metadata.uid` is the only identity that fences the OAuth+ * teardown and the Secret cleanup against a same-name reinstall. While the CR is+ * alive a failed step can be retried with the same uid, and a reinstall through an+ * install route cannot create its CR (the name is taken), so it cannot race the+ * Context strip either. Deleting the CR first — the previous order — made every+ * cleanup failure permanent: the retry found no CR, hence no uid, hence no teardown.+ *+ * 1. read the CR → uid + Secret snapshots (404 → `not_found`, nothing touched)+ * 2. strip the name from every Context allowlist (→ `contexts`)+ * 3. OAuth teardown fenced by the uid (→ `dynamic_client` | `oauth_grants`)+ * 4. delete the snapshotted Secrets (→ `secrets`)+ * 5. delete the CR fenced by the uid (→ `mcp_server`)+ * 6. second grants purge by the uid — sweeps a grant a consent callback sealed with+ * this uid between 3 and 5; a failure only counts and logs, because readers+ * fenced on the live CR uid already treat such a row as inert.+ *+ * The first failing step stops the run and is reported as `incomplete` with the CR+ * still present, so repeating the same request completes it. The result is transport+ * neutral: each route maps it to its own HTTP contract.+ */++export type McpServerUninstallStage =+ | 'contexts'+ | 'dynamic_client'+ | 'oauth_grants'+ | 'secrets'+ | 'mcp_server'++export type McpServerUninstallResult =+ | { status: 'not_found' }+ | { status: 'completed'; crUid: string; deleted: string[]; crDeleteResponse: unknown }+ | { status: 'incomplete'; pending: McpServerUninstallStage[]; deleted: string[] }++export type UninstallGateway = Pick<+ K8sGateway,+ | 'getResource'+ | 'listResource'+ | 'updateResource'+ | 'deleteResource'+ | 'getSecret'+ | 'deleteSecret'+>++export interface McpServerUninstallDeps {+ gateway: UninstallGateway+ db: DbClient+ encryptionKey: Buffer+ dcrDeps: DcrDeps+ /** Contexts live in their own namespace, never in the McpServer's. */+ contextsNamespace: string+ logger: Logger+ /**+ * Awaited after each accepted K8s delete, before the step counts as done. The+ * apiserver accepts a delete before finalizers run; a caller that must not report+ * success until the object is gone (registry) blocks here.+ */+ awaitDeletion?: (target: {+ kind: 'McpServer' | 'Secret'+ name: string+ namespace: string+ }) => Promise<void>+}++/** Both dependent Secrets of an McpServer, in their fixed cleanup order. */+export async function captureMcpServerSecrets(+ gateway: Pick<K8sGateway, 'getSecret'>,+ name: string,+ namespace: string,+ logger: Logger+): Promise<SecretCleanupCapture[]> {+ return [+ await captureSecretForCleanup(gateway, `${name}-credentials`, namespace, logger),+ // Pre-registered confidential remote installs create this Secret before the CR,+ // with no ownerReferences, so K8s GC never reaps it.+ await captureSecretForCleanup(gateway, `${name}-oauth-client`, namespace, logger, {+ requireManagedOwnership: true,+ }),+ ]+}++/**+ * Deletes every `ready` capture, fenced on its snapshot. A 404 is already clean.+ * Returns the names deleted and whether any delete failed; never throws.+ */+export async function deleteCapturedSecrets(+ gateway: Pick<K8sGateway, 'deleteSecret'>,+ captures: SecretCleanupCapture[],+ namespace: string,+ logger: Logger,+ awaitDeletion?: McpServerUninstallDeps['awaitDeletion']+): Promise<{ deleted: string[]; failed: boolean }> {+ const deleted: string[] = []+ let failed = false+ for (const capture of captures) {+ if (capture.status !== 'ready') {+ if (capture.status !== 'absent') {+ logger.warn(+ { secretName: capture.name, namespace, reason: capture.status },+ 'Skipped McpServer Secret cleanup'+ )+ }+ continue+ }+ try {+ await gateway.deleteSecret(capture.name, namespace, capture.precondition)+ await awaitDeletion?.({ kind: 'Secret', name: capture.name, namespace })+ deleted.push(`Secret/${capture.name}`)+ logger.info({ secretName: capture.name, namespace }, 'Deleted McpServer Secret')+ } catch (err) {+ if (extractK8sError(err)?.status === 404) {+ logger.info({ secretName: capture.name, namespace }, 'McpServer Secret already gone')+ continue+ }+ failed = true+ mcpServerUninstallTeardownFailuresTotal.inc({+ stage: capture.name.endsWith('-oauth-client') ? 'oauth_client_secret' : 'secrets',+ })+ logger.error(+ { secretName: capture.name, namespace, err },+ 'McpServer Secret cleanup failed on uninstall'+ )+ }+ }+ return { deleted, failed }+}++function readCrUid(cr: unknown): string | undefined {+ const uid = (cr as { metadata?: { uid?: unknown } } | null)?.metadata?.uid+ return typeof uid === 'string' && uid ? uid : undefined+}++export async function uninstallMcpServer(+ deps: McpServerUninstallDeps,+ target: { name: string; namespace: string }+): Promise<McpServerUninstallResult> {+ const { gateway, logger } = deps+ const { name, namespace } = target+ const deleted: string[] = []+ const incomplete = (stage: McpServerUninstallStage[]): McpServerUninstallResult => ({+ status: 'incomplete',+ pending: stage,+ deleted,+ })++ // 1 · CR identity + Secret snapshots.+ let cr: unknown+ try {+ cr = await gateway.getResource('mcpservers', name, namespace)+ } catch (err) {+ if (isNotFound(err)) return { status: 'not_found' }+ throw err+ }+ const crUid = readCrUid(cr)+ if (!crUid) {+ // Every cleanup below is fenced by this uid, so without it nothing can run safely.+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'mcp_server' })+ logger.error(+ { serverName: name, namespace, reason: 'cr_uid_missing' },+ 'McpServer read returned no metadata.uid; a real apiserver always sets it, so this ' ++ 'points at a non-apiserver gateway or a broken proxy — retrying will not resolve it'+ )+ return incomplete(['mcp_server'])+ }+ const secretCaptures = await captureMcpServerSecrets(gateway, name, namespace, logger)+ // An unreadable Secret now would be orphaned once the CR is gone; stop before any+ // mutation so the retry re-captures it.+ if (secretCaptures.some(c => c.status === 'read-failed')) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'secrets' })+ return incomplete(['secrets'])+ }++ // 2 · Context allowlists.+ try {+ await stripServerFromContexts(gateway, deps.contextsNamespace, name, contextName => {+ deleted.push(`Context/${contextName} (removed from allowlist)`)+ logger.info({ serverName: name, contextName }, 'Removed MCP server from Context allowlist')+ })+ } catch (err) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'contexts' })+ logger.error({ serverName: name, err }, 'Context allowlist cleanup failed on uninstall')+ return incomplete(['contexts'])+ }++ // 3 · OAuth state of THIS installation.+ const teardown = await teardownMcpServerOAuthState(+ deps.db,+ deps.encryptionKey,+ deps.dcrDeps,+ { namespace, name, crUid },+ logger+ )+ if (teardown.dynamicClient === 'done') deleted.push(`DynamicClient/${name}`)+ if (teardown.grants === 'done') deleted.push(`OAuthGrants/${name}`)+ const oauthPending: McpServerUninstallStage[] = []+ if (teardown.dynamicClient === 'failed') oauthPending.push('dynamic_client')+ if (teardown.grants === 'failed') oauthPending.push('oauth_grants')+ if (oauthPending.length > 0) return incomplete(oauthPending)++ // 4 · Secrets, fenced on the step-1 snapshots.+ const secrets = await deleteCapturedSecrets(+ gateway,+ secretCaptures,+ namespace,+ logger,+ deps.awaitDeletion+ )+ deleted.push(...secrets.deleted)+ if (secrets.failed) return incomplete(['secrets'])++ // 5 · The CR itself, fenced on the uid every cleanup above was bound to.+ let crDeleteResponse: unknown+ try {+ crDeleteResponse = await gateway.deleteResource('mcpservers', name, namespace, { uid: crUid })+ await deps.awaitDeletion?.({ kind: 'McpServer', name, namespace })+ } catch (err) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'mcp_server' })+ logger.error({ serverName: name, namespace, err }, 'McpServer delete failed on uninstall')+ return incomplete(['mcp_server'])+ }+ deleted.push(`McpServer/${name}`)++ // 6 · Late-sealed grants of this uid.+ try {+ const purged = await deleteOAuthGrantsForServer(deps.db, {+ recipeNamespace: namespace,+ recipeName: name,+ crUid,+ })+ if (purged > 0) {+ logger.info(+ { serverName: name, namespace, count: purged },+ 'Purged OAuth grants sealed after the uninstall teardown'+ )+ }+ } catch (err) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants_post_delete' })+ logger.error(+ { serverName: name, namespace, err },+ 'Post-delete OAuth grants purge failed on uninstall'+ )+ }++ return { status: 'completed', crUid, deleted, crDeleteResponse }+}diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 0563c285e..cbfebc4b2 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -189,14 +189,16 @@ export const mcpHostJwtReissueTotal = getOrCreateCounter({labelNames: ['result'] as const as Array<'result'>,})-// MCP server uninstall teardown: a best-effort cleanup step failed AFTER the CR was-// already deleted (orphan oauth-client Secret, dynamic client row, or oauth_grants).-// Nothing retries once the CR is gone, so the structured error log is the only trace-// today — this counter makes the residual reconcilable/alertable. Rate > 0 ⇒ orphaned-// state to sweep. Stages: oauth_client_secret | dynamic_client | oauth_grants.+// MCP server uninstall: a cleanup step failed. Every stage except+// oauth_grants_post_delete runs before the CR delete and leaves the CR in place for a+// retry (the request answers "repair required"); a sustained rate means uninstalls+// that nobody repaired. oauth_grants_post_delete is the only residual after a 200 —+// rows it leaves are inert to the uid-fenced readers. Stages: contexts |+// dynamic_client | oauth_grants | secrets | oauth_client_secret | mcp_server |+// oauth_grants_post_delete.export const mcpServerUninstallTeardownFailuresTotal = getOrCreateCounter({name: 'mcp_server_uninstall_teardown_failures_total',- help: 'Count of best-effort MCP server uninstall teardown steps that failed after CR deletion, by stage.',+ help: 'Count of MCP server uninstall cleanup steps that failed, by stage.',labelNames: ['stage'] as const as Array<'stage'>,})diff --git a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.tsindex 54c554344..74676de36 100644--- a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts+++ b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts@@ -3,10 +3,10 @@ import express from 'express'import request from 'supertest'/**- * R2-L2 (T3/T4): a best-effort uninstall teardown step that fails AFTER the CR is- * deleted increments mcp_server_uninstall_teardown_failures_total{stage} — the- * signal that makes orphaned Secret / dynamic-client / grants state reconcilable- * and alertable. The uninstall still returns 200: teardown never blocks the delete.+ * R2-L2 (T3/T4): a failed uninstall teardown step increments+ * mcp_server_uninstall_teardown_failures_total{stage}. The teardown now runs BEFORE+ * the CR delete, so the failure also stops the uninstall: 503 repair_required with+ * the CR still present (a 200 here used to mean "CR gone, grants orphaned for good").** Seam: the same pool.query swap as mcpServerUninstallPurgesGrants.test.ts, but the* oauth_grants DELETE throws so the H-3 catch runs. The counter is read from the@@ -33,6 +33,8 @@ vi.mock('../../../db.js', async importActual => {// Import AFTER vi.mock so the swapped `pool` wires into the router.const { createAdminResourcesRouter } = await import('../resources.js')const { registry } = await import('../../../observability/metrics.js')+const { config } = await import('../../../config.js')+const { MockGateway } = await import('../../../../test/mockGateway.js')async function teardownFailures(stage: string): Promise<number> {const all = await registry.getMetricsAsJSON()@@ -46,37 +48,33 @@ async function teardownFailures(stage: string): Promise<number> {describe('DELETE /admin/mcp-servers/:name — teardown failure increments the counter (R2-L2)', () => {const SERVER_NAME = 'gdrive'- function buildApp() {- const gateway = {- // The uninstall reads the CR before deleting to capture metadata.uid (R3-H5).- getResource: vi.fn(async () => ({- metadata: { name: SERVER_NAME, uid: 'uid-gdrive-live', resourceVersion: '1' },- })),- deleteResource: vi.fn(async () => ({ metadata: { name: SERVER_NAME } })),- deleteSecret: vi.fn(async () => ({})),- getSecret: vi.fn(async () => ({})),- listResource: vi.fn(async () => []),- updateResource: vi.fn(async () => ({})),- }+ async function buildApp() {+ const NS = config.mcpServersNamespace+ const gateway = new MockGateway(NS)+ await gateway.createResource('mcpservers', { metadata: { name: SERVER_NAME }, spec: {} }, NS)const app = express()app.use(express.json({ limit: '1mb' }))app.use('/api/v1', createAdminResourcesRouter(gateway as never))- return { app }+ return { app, gateway, NS }}beforeEach(() => {dbMock.query.mockClear()})- it('counts a failed oauth_grants purge under stage=oauth_grants and still returns 200', async () => {+ it('counts a failed oauth_grants purge under stage=oauth_grants and keeps the CR for a retry', async () => {const before = await teardownFailures('oauth_grants')- const { app } = buildApp()+ const { app, gateway, NS } = await buildApp()const res = await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)- // Teardown is best-effort — a purge failure must never block the uninstall.- expect(res.status).toBe(200)- // The failed stage is now an observable metric, not just a log line.+ expect(res.status).toBe(503)+ expect(res.body).toMatchObject({+ error: 'mcp_server_uninstall_incomplete',+ outcome: 'repair_required',+ pending: ['oauth_grants'],+ })+ await expect(gateway.getResource('mcpservers', SERVER_NAME, NS)).resolves.toBeTruthy()const after = await teardownFailures('oauth_grants')expect(after - before).toBeGreaterThanOrEqual(1)})diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex b754cae78..bb636e37a 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -15,6 +15,7 @@ import { GenericConfigSuggestionSchema } from '../../oauth/genericKnobs.js'import { getOAuthProviderAdapter, isKnownOAuthProvider } from '../../oauth/providers.js'import { rootLogger } from '../../observability/logger.js'import { type AdminUserRecord, findAdminById } from '../../services/adminAuthService.js'+import { attachServerToContext } from '../../services/contextAllowlist.js'import {addHookRefToHost,listHostsReferencingHook,@@ -2726,42 +2727,18 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {const contextsNs = config.contextsNamespacelet contextBefore: RegistryResourceSnapshot | null = nulltry {- const ctx = (await gateway.getResource('contexts', contextRef, contextsNs)) as {- metadata?: { uid?: string; resourceVersion?: string }- spec?: Record<string, unknown> & {- contextId?: string- description?: string- mcpServers?: string[]- }- }- contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, contextsNs)- if (!contextBefore.metadata?.uid || !contextBefore.metadata.resourceVersion) {- throw Object.assign(new Error('Context identity is unavailable'), {- statusCode: 503,- code: 'context_identity_unavailable',- })- }- const existing: string[] = ctx.spec?.mcpServers ?? []- if (!existing.includes(serverName)) {- await gateway.updateResource(- 'contexts',- contextRef,- {- metadata: {- uid: contextBefore.metadata.uid,- ...(ctx.metadata?.resourceVersion- ? { resourceVersion: ctx.metadata.resourceVersion }- : {}),- },- spec: {- ...ctx.spec,- contextId: ctx.spec?.contextId ?? contextRef,- mcpServers: [...existing, serverName],- } as Record<string, unknown>,+ // The before-image for the ambiguous-outcome readback below is the snapshot+ // the LAST attempt edited: an earlier one is stale after a 409 retry.+ await attachServerToContext(+ gateway,+ { name: contextRef, namespace: contextsNs },+ serverName,+ {+ onRead: ctx => {+ contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, contextsNs)},- contextsNs- )- }+ }+ )} catch (err) {let associationOutcome: RegistryMutationOutcome = isDeterministicRegistryNoCommit(err)? 'not-committed'diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.tsindex 81d99900a..06307fb27 100644--- a/control-api/src/routes/admin/remoteMcp.ts+++ b/control-api/src/routes/admin/remoteMcp.ts@@ -34,6 +34,7 @@ import {import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'import { probeMcpTransport } from '../../oauth/mcpTransportProbe.js'import { rootLogger } from '../../observability/logger.js'+import { attachServerToContext } from '../../services/contextAllowlist.js'import { K8sNotFoundError } from '../../services/resourceService.js'import type { SecretSnapshot } from '../../services/secretRepository.js'import type { ResourcePreconditions } from '../../types.js'@@ -1111,19 +1112,7 @@ export function createAdminRemoteMcpRouter(// ── Saga step 3: attach to the Context allowlist (rollback CR+Secret) ──try {- const ctx = (await gateway.getResource('contexts', contextRef)) as {- spec?: Record<string, unknown> & { contextId?: string; mcpServers?: string[] }- }- const existing: string[] = ctx.spec?.mcpServers ?? []- if (!existing.includes(serverName)) {- await gateway.updateResource('contexts', contextRef, {- spec: {- ...ctx.spec,- contextId: ctx.spec?.contextId ?? contextRef,- mcpServers: [...existing, serverName],- } as Record<string, unknown>,- })- }+ await attachServerToContext(gateway, { name: contextRef }, serverName)} catch (err) {try {await gateway.deleteResource(diff --git a/control-api/src/routes/admin/resources.ts b/control-api/src/routes/admin/resources.tsindex 9cc8aa66f..a888a0397 100644--- a/control-api/src/routes/admin/resources.ts+++ b/control-api/src/routes/admin/resources.ts@@ -13,21 +13,21 @@ import { validateCommunicationChannelSpec } from '../../http/validateCommunicatiimport { validateMcpServerSpecPreflight } from '../../http/validateMcpServerSpec.js'import { K8sGateway } from '../../k8s.js'import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'-import { teardownMcpServerOAuthState } from '../../oauth/mcpServerOAuthTeardown.js'+import {+ captureMcpServerSecrets,+ deleteCapturedSecrets,+ uninstallMcpServer,+} from '../../oauth/mcpServerUninstall.js'import { rootLogger } from '../../observability/logger.js'-import { mcpServerUninstallTeardownFailuresTotal } from '../../observability/metrics.js'+import { stripServerFromContexts } from '../../services/contextAllowlist.js'import { stripHookRefFromHosts } from '../../services/hostGuardrailRefs.js'import {K8sConflictError,K8sNotFoundError,type MutableResourceSnapshot,} from '../../services/resourceService.js'+import { captureSecretForCleanup } from '../../services/secretCleanup.js'import { secretKeyNames } from '../../services/secretKeyNames.js'-import {- isControlApiManagedSecret,- isRecipeOwnedSecret,- secretIdentityPreconditions,-} from '../../services/secretRepository.js'import {ClerumResourceType,type ResourcePreconditions,@@ -1079,6 +1079,47 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {const { plural, ns } = resolveResource(req.params.resource)!const name = req.params.name+ if (plural === 'mcpservers') {+ const outcome = await uninstallMcpServer(+ {+ gateway,+ db: dcrDb,+ encryptionKey: oauthEncryptionKey,+ dcrDeps: { logger: log },+ contextsNamespace: resourceNamespace('contexts'),+ logger: log,+ },+ { name, namespace: ns }+ )+ if (outcome.status === 'completed') {+ res.status(200).json(outcome.crDeleteResponse)+ return+ }+ if (outcome.status === 'incomplete') {+ res.status(503).json({+ error: 'mcp_server_uninstall_incomplete',+ outcome: 'repair_required',+ pending: outcome.pending,+ deleted: outcome.deleted,+ })+ return+ }+ // The CR read found nothing, so there is no uid to fence an OAuth teardown by:+ // only the name-addressed K8s cleanup runs, and OAuth state is left alone.+ const secretCaptures = await captureMcpServerSecrets(gateway, name, ns, log)+ const deleted = await gateway.deleteResource(plural, name, ns)+ await deleteCapturedSecrets(gateway, secretCaptures, ns, log)+ try {+ await stripServerFromContexts(gateway, resourceNamespace('contexts'), name, contextName =>+ log.info({ serverName: name, contextName }, 'Removed MCP server from Context allowlist')+ )+ } catch (err) {+ log.error({ serverName: name, err }, 'Failed to clean up Context allowlists')+ }+ res.status(200).json(deleted)+ return+ }+// CommunicationChannel: read the credentialsSecretRef name FIRST (we// can't read the CC after it's deleted), then delete the CC, then// delete the Secret. The CC-first order means HCC's SecretInformer@@ -1102,105 +1143,15 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {}}- // Capture the credential Secret identity BEFORE deleting the parent CR.- // A same-name Secret can be created in the gap, and a name-addressed- // delete afterwards would remove that replacement — a different owner's- // object — on the strength of a decision made about something that no- // longer exists. Every cleanup below is bound to the snapshot taken here.- // Same reasoning, and same shape, as the registry uninstall path.- const captureSecretForCleanup = async (- secretName: string,- { requireManagedOwnership = false }: { requireManagedOwnership?: boolean } = {}- ): Promise<- | { status: 'ready'; precondition: SecretPreconditions }- | {- status:- | 'absent'- | 'recipe-owned'- | 'not-managed'- | 'identity-unavailable'- | 'read-failed'- }- > => {- let raw: unknown- try {- raw = await gateway.getSecret(secretName, ns)- } catch (err) {- if (extractK8sStatusCode(err) === 404) return { status: 'absent' }- // Deleting the CR is the request's actual intent; the Secret is a- // cascade. A failed read means we cannot fence the cascade delete, so- // it is skipped and logged — the same policy this PR already applies- // to the rollback paths ("rollback skipped: identity unavailable").- // Failing the whole request here would leave the CR undeleted because- // its dependent Secret could not be read.- log.error(- { secretName, namespace: ns, err },- 'Secret cleanup capture failed; cascade delete will be skipped'- )- return { status: 'read-failed' }- }- // Recipe-owned Secrets belong to /admin/recipe-secrets. The mcp-secret- // route refuses them with 409; deleting one here would route around- // that guard.- if (isRecipeOwnedSecret(raw)) return { status: 'recipe-owned' }- // Derived-name Secrets (`${name}-oauth-client`) share a name with any- // operator-owned Secret referenced in OAuth reference mode. Only delete- // when the managed-by label proves control-api's install saga created it;- // UID/RV fencing guarantees same-object, not same-owner.- if (requireManagedOwnership && !isControlApiManagedSecret(raw)) {- return { status: 'not-managed' }- }- const precondition = secretIdentityPreconditions(raw)- if (!precondition) return { status: 'identity-unavailable' }- return { status: 'ready', precondition }- }-+ // Capture the credential Secret identity BEFORE deleting the parent CR: a+ // same-name Secret created in the gap belongs to a different owner, and a+ // name-addressed delete afterwards would remove it.const ccSecretCleanup =plural === 'communicationchannels' && ccSecretRefName- ? await captureSecretForCleanup(ccSecretRefName)- : null- const mcpCredentialsSecretName = `${name}-credentials`- const mcpSecretCleanup =- plural === 'mcpservers' ? await captureSecretForCleanup(mcpCredentialsSecretName) : null- // Pre-registered confidential remote installs create a `${name}-oauth-client`- // Secret in the install saga with NO ownerReferences (it predates the CR), so- // K8s GC never reaps it — control-api must delete it here. Fence it on the- // identity captured BEFORE the CR delete, exactly like the credentials Secret:- // a name-addressed delete could otherwise clobber a fresh same-name Secret- // written by a concurrent reinstall in the gap (the bug this capture shape- // exists to prevent). 'absent' (404) is the norm for non-pre-registered servers.- const mcpOAuthClientSecretName = `${name}-oauth-client`- const mcpOAuthClientSecretCleanup =- plural === 'mcpservers'- ? await captureSecretForCleanup(mcpOAuthClientSecretName, {- requireManagedOwnership: true,- })+ ? await captureSecretForCleanup(gateway, ccSecretRefName, ns, log): null- // mcpservers only: read the CR before deleting so its metadata.uid fences BOTH- // the delete (a same-name reinstall's CR has a different uid → the delete- // precondition 409s and we never tear down its state) AND the OAuth teardown- // (only this installation's dynamic_clients row + grants are removed, R3-H5). A- // 404 means the CR is already gone → no uid, so the teardown is skipped entirely.- let mcpServerCrUid: string | undefined- if (plural === 'mcpservers') {- try {- const cr = await gateway.getResource(plural, name, ns)- mcpServerCrUid = resourcePreconditionsFromResource(cr)?.uid- } catch (err) {- // getResource wraps a namespaced 404 as K8sNotFoundError (httpStatus, not- // statusCode), so match both shapes — as the update handler does above.- if (!(err instanceof K8sNotFoundError) && extractK8sStatusCode(err) !== 404) throw err- // CR already gone: leave uid undefined so the teardown below is skipped.- }- }-- // Only the mcpservers delete is uid-fenced; every other plural keeps the exact- // 3-arg call it had (a trailing undefined would change the observed call shape).- const deleted =- mcpServerCrUid !== undefined- ? await gateway.deleteResource(plural, name, ns, { uid: mcpServerCrUid })- : await gateway.deleteResource(plural, name, ns)+ const deleted = await gateway.deleteResource(plural, name, ns)if (plural === 'communicationchannels' && ccSecretRefName && ccSecretCleanup) {if (ccSecretCleanup.status !== 'ready') {@@ -1233,153 +1184,6 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {}}- if (plural === 'mcpservers') {- const contextsNs = resourceNamespace('contexts')- if (mcpSecretCleanup && mcpSecretCleanup.status !== 'ready') {- log.warn(- {- secretName: mcpCredentialsSecretName,- namespace: ns,- reason: mcpSecretCleanup.status,- },- 'Skipped MCP credentials Secret cleanup'- )- } else if (mcpSecretCleanup) {- try {- await gateway.deleteSecret(mcpCredentialsSecretName, ns, mcpSecretCleanup.precondition)- log.info(- { secretName: mcpCredentialsSecretName, namespace: ns },- 'Deleted MCP credentials Secret'- )- } catch (err) {- if (extractK8sStatusCode(err) === 404) {- log.info(- { secretName: mcpCredentialsSecretName, namespace: ns },- 'MCP credentials Secret already gone'- )- } else {- // The McpServer is already gone, so a 200 is still the honest- // outcome — but this is a cleanup FAILURE, not an absence. The- // previous catch reported every error class, 403 included, as- // "no Secret to delete", telling the operator a credential had- // been removed while it was still live.- log.error(- { secretName: mcpCredentialsSecretName, namespace: ns, err },- 'McpServer delete succeeded but credentials cleanup failed'- )- }- }- }-- // Pre-registered confidential remote installs leave a `${name}-oauth-client`- // Secret with no ownerReferences (see the capture above). Delete it fenced on- // the identity captured before the CR delete; 'absent' (404) is expected for- // every non-pre-registered server and never fails the uninstall.- if (mcpOAuthClientSecretCleanup) {- if (mcpOAuthClientSecretCleanup.status === 'ready') {- try {- await gateway.deleteSecret(- mcpOAuthClientSecretName,- ns,- mcpOAuthClientSecretCleanup.precondition- )- log.info(- { secretName: mcpOAuthClientSecretName, namespace: ns },- 'Deleted MCP OAuth client Secret'- )- } catch (err) {- if (extractK8sStatusCode(err) === 404) {- log.info(- { secretName: mcpOAuthClientSecretName, namespace: ns },- 'MCP OAuth client Secret already gone'- )- } else {- mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_client_secret' })- log.error(- { secretName: mcpOAuthClientSecretName, namespace: ns, err },- 'McpServer delete succeeded but OAuth client Secret cleanup failed'- )- }- }- } else if (mcpOAuthClientSecretCleanup.status === 'absent') {- // Expected for every non-pre-registered server: no OAuth client Secret exists.- log.info(- { secretName: mcpOAuthClientSecretName, namespace: ns },- 'No MCP OAuth client Secret to delete'- )- } else {- log.warn(- {- secretName: mcpOAuthClientSecretName,- namespace: ns,- reason: mcpOAuthClientSecretCleanup.status,- },- 'Skipped MCP OAuth client Secret cleanup'- )- }- }-- // OAuth teardown fenced by the CR uid (R3-H5): revoke this installation's- // encrypted `dynamic_clients` row + best-effort RFC 7592 delete at the AS, and- // purge its `oauth_grants` rows — never a same-name reinstall's state. Skipped- // when the uid is unknown (the CR read 404'd): a name-only teardown here would- // reopen the very race this fences. The module never throws; it counts/logs per- // stage and the uninstall keeps its 200 (the response policy is R3-H7).- if (mcpServerCrUid !== undefined) {- const teardown = await teardownMcpServerOAuthState(- dcrDb,- oauthEncryptionKey,- { logger: log },- { namespace: ns, name, crUid: mcpServerCrUid },- log- )- log.info(- {- serverName: name,- namespace: ns,- dynamicClient: teardown.dynamicClient,- grants: teardown.grants,- },- 'MCP server OAuth teardown completed'- )- }-- try {- const ctxList = (await gateway.listResource('contexts', contextsNs)) as Array<{- metadata?: { name?: string }- spec?: Record<string, unknown> & {- contextId?: string- description?: string- mcpServers?: string[]- }- }>- for (const ctx of ctxList) {- const ctxName = ctx.metadata?.name- const servers = ctx.spec?.mcpServers ?? []- if (ctxName && servers.includes(name)) {- await gateway.updateResource(- 'contexts',- ctxName,- {- spec: {- ...ctx.spec,- contextId: ctx.spec?.contextId ?? ctxName,- mcpServers: servers.filter(s => s !== name),- } as Record<string, unknown>,- },- contextsNs- )- log.info(- { serverName: name, contextName: ctxName },- 'Removed MCP server from Context allowlist'- )- }- }- } catch (err) {- log.error({ serverName: name, err }, 'Failed to clean up Context allowlists')- }- }-res.status(200).json(deleted)}))diff --git a/control-api/src/services/contextAllowlist.ts b/control-api/src/services/contextAllowlist.tsnew file mode 100644index 000000000..4920126a0--- /dev/null+++ b/control-api/src/services/contextAllowlist.ts@@ -0,0 +1,156 @@+import { extractK8sError } from '../http/k8sError.js'+import type { K8sGateway } from '../k8s.js'+import { K8sNotFoundError } from './resourceService.js'++/** Shape of a Context CR as the allowlist writers read it. */+export type ContextAllowlistSnapshot = {+ metadata?: { name?: string; uid?: string; resourceVersion?: string }+ spec?: Record<string, unknown> & { contextId?: string; mcpServers?: string[] }+}++/**+ * Pure edit of `spec.mcpServers`. Returns the new list, or `null` when the current+ * list already satisfies the caller (no write is issued).+ */+export type ContextAllowlistEdit = (servers: readonly string[]) => string[] | null++/**+ * Same shape as the registry's historical inline error so its outcome classifier+ * (`statusCode` 503, string `code`) keeps treating it identically.+ */+export class ContextIdentityUnavailableError extends Error {+ readonly statusCode = 503+ readonly code = 'context_identity_unavailable'+ constructor(contextName: string) {+ super(`Context/${contextName} identity unavailable; refusing stale update`)+ this.name = 'ContextIdentityUnavailableError'+ }+}++/**+ * The Context status subresource moves the resourceVersion on every controller status+ * write, so a first-attempt 409 is routine and not evidence of a competing allowlist+ * writer. Three attempts absorb that churn while still surfacing a genuinely hot object.+ */+export const CONTEXT_ALLOWLIST_MAX_ATTEMPTS = 3++type AllowlistGateway = Pick<K8sGateway, 'getResource' | 'updateResource'>++function isConflict(err: unknown): boolean {+ return extractK8sError(err)?.status === 409+}++export function isNotFound(err: unknown): boolean {+ return err instanceof K8sNotFoundError || extractK8sError(err)?.status === 404+}++/**+ * Read-edit-write of one Context's `spec.mcpServers`, fenced on the uid AND+ * resourceVersion of the read it edited. A write without the resourceVersion replays+ * a stale list: an attach that read `[X]` before X's uninstall stripped it would put+ * X back. On 409 the Context is re-read and the edit recomputed from the fresh list,+ * never replayed.+ *+ * `initial` lets a caller that already listed the Context skip the first GET.+ * `onRead` observes every snapshot the edit is computed from (registry keeps the last+ * one as the before-image for its ambiguous-outcome readback).+ */+export async function editContextAllowlist(+ gateway: AllowlistGateway,+ target: { name: string; namespace?: string },+ edit: ContextAllowlistEdit,+ options: {+ initial?: ContextAllowlistSnapshot+ onRead?: (ctx: ContextAllowlistSnapshot) => void+ maxAttempts?: number+ } = {}+): Promise<{ changed: boolean }> {+ const { name, namespace } = target+ const maxAttempts = options.maxAttempts ?? CONTEXT_ALLOWLIST_MAX_ATTEMPTS+ for (let attempt = 1; ; attempt += 1) {+ const ctx =+ attempt === 1 && options.initial+ ? options.initial+ : ((namespace === undefined+ ? await gateway.getResource('contexts', name)+ : await gateway.getResource('contexts', name, namespace)) as ContextAllowlistSnapshot)+ options.onRead?.(ctx)+ const uid = ctx.metadata?.uid+ const resourceVersion = ctx.metadata?.resourceVersion+ if (+ typeof uid !== 'string' ||+ !uid ||+ typeof resourceVersion !== 'string' ||+ !resourceVersion+ ) {+ throw new ContextIdentityUnavailableError(name)+ }+ const next = edit(ctx.spec?.mcpServers ?? [])+ if (next === null) return { changed: false }+ const body = {+ metadata: { uid, resourceVersion },+ spec: {+ ...ctx.spec,+ contextId: ctx.spec?.contextId ?? name,+ mcpServers: next,+ } as Record<string, unknown>,+ }+ try {+ if (namespace === undefined) {+ await gateway.updateResource('contexts', name, body)+ } else {+ await gateway.updateResource('contexts', name, body, namespace)+ }+ return { changed: true }+ } catch (err) {+ if (isConflict(err) && attempt < maxAttempts) continue+ throw err+ }+ }+}++/** Adds `serverName` to the Context allowlist unless it is already there. */+export function attachServerToContext(+ gateway: AllowlistGateway,+ target: { name: string; namespace?: string },+ serverName: string,+ options: { onRead?: (ctx: ContextAllowlistSnapshot) => void } = {}+): Promise<{ changed: boolean }> {+ return editContextAllowlist(+ gateway,+ target,+ servers => (servers.includes(serverName) ? null : [...servers, serverName]),+ options+ )+}++/**+ * Removes `serverName` from every Context in `namespace` that allowlists it. A Context+ * deleted between the list and its write is already clean. Throws on the first Context+ * that cannot be written; the ones already stripped are returned through `onStripped`+ * so a partial run is reportable.+ */+export async function stripServerFromContexts(+ gateway: Pick<K8sGateway, 'listResource' | 'getResource' | 'updateResource'>,+ namespace: string,+ serverName: string,+ onStripped: (contextName: string) => void+): Promise<void> {+ const contexts = (await gateway.listResource('contexts', namespace)) as ContextAllowlistSnapshot[]+ for (const ctx of contexts) {+ const contextName = ctx.metadata?.name+ if (!contextName || !(ctx.spec?.mcpServers ?? []).includes(serverName)) continue+ try {+ const { changed } = await editContextAllowlist(+ gateway,+ { name: contextName, namespace },+ servers => (servers.includes(serverName) ? servers.filter(s => s !== serverName) : null),+ { initial: ctx }+ )+ if (changed) onStripped(contextName)+ } catch (err) {+ if (isNotFound(err)) continue+ throw err+ }+ }+}diff --git a/control-api/src/services/secretCleanup.ts b/control-api/src/services/secretCleanup.tsnew file mode 100644index 000000000..5746ace1d--- /dev/null+++ b/control-api/src/services/secretCleanup.ts@@ -0,0 +1,49 @@+import { extractK8sError } from '../http/k8sError.js'+import type { K8sGateway } from '../k8s.js'+import type { Logger } from '../observability/logger.js'+import type { SecretPreconditions } from '../types.js'+import {+ isControlApiManagedSecret,+ isRecipeOwnedSecret,+ secretIdentityPreconditions,+} from './secretRepository.js'++export type SecretCleanupCapture =+ | { status: 'ready'; name: string; precondition: SecretPreconditions }+ | {+ status: 'absent' | 'recipe-owned' | 'not-managed' | 'identity-unavailable' | 'read-failed'+ name: string+ }++/**+ * Snapshot a dependent Secret's identity so its later delete is bound to this exact+ * object: a same-name Secret written afterwards is a different owner's.+ * `requireManagedOwnership` guards derived names (`${name}-oauth-client`) that OAuth+ * reference mode lets an operator Secret share — uid/RV proves same object, not same+ * owner, so only a Secret labelled as created by control-api's install saga qualifies.+ */+export async function captureSecretForCleanup(+ gateway: Pick<K8sGateway, 'getSecret'>,+ secretName: string,+ namespace: string,+ logger: Logger,+ { requireManagedOwnership = false }: { requireManagedOwnership?: boolean } = {}+): Promise<SecretCleanupCapture> {+ let raw: unknown+ try {+ raw = await gateway.getSecret(secretName, namespace)+ } catch (err) {+ if (extractK8sError(err)?.status === 404) return { status: 'absent', name: secretName }+ logger.error({ secretName, namespace, err }, 'Secret cleanup capture failed')+ return { status: 'read-failed', name: secretName }+ }+ // Recipe-owned Secrets belong to /admin/recipe-secrets, whose guard a cascade+ // delete here would route around.+ if (isRecipeOwnedSecret(raw)) return { status: 'recipe-owned', name: secretName }+ if (requireManagedOwnership && !isControlApiManagedSecret(raw)) {+ return { status: 'not-managed', name: secretName }+ }+ const precondition = secretIdentityPreconditions(raw)+ if (!precondition) return { status: 'identity-unavailable', name: secretName }+ return { status: 'ready', name: secretName, precondition }+}diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.tsindex 6bd833579..2c9814674 100644--- a/control-api/test/routes.adminRemoteMcp.test.ts+++ b/control-api/test/routes.adminRemoteMcp.test.ts@@ -580,7 +580,10 @@ describe('POST /admin/mcp-servers/remote (install saga)', () => {NS)) as { metadata: { uid: string } }recreatedCrUid = recreated.metadata.uid- throw Object.assign(new Error('conflict'), { code: 409 })+ // Not a 409: the attach re-reads and retries on a conflict (the Context status+ // subresource moves its resourceVersion), so only a non-retryable failure+ // reaches the compensation this test is about.+ throw Object.assign(new Error('context write failed'), { code: 500 })})const res = await request(makeApp(gw)).post('/admin/mcp-servers/remote').send({diff --git a/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.ts b/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.tsnew file mode 100644index 000000000..d6ef16f46--- /dev/null+++ b/control-api/test/routes.mcpServerUninstall.contextAllowlist.test.ts@@ -0,0 +1,190 @@+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import type { PinnedTransport } from '../src/http/pinnedFetch.js'+import type { K8sGateway } from '../src/k8s.js'+import {+ type DiscoveryOutcome,+ type DiscoveryResult,+ discoverRemoteOAuth,+} from '../src/oauth/discovery.js'+import { probeMcpTransport } from '../src/oauth/mcpTransportProbe.js'+import { PILOTS, makeDiscoveryTransport } from './fixtures/remoteOAuthDiscovery.js'+import { MockGateway } from './mockGateway.js'++/**+ * The McpServer uninstall vs. the Context allowlist writers of the install routes+ * (R3-H5b + R3-S6), observed as the FINAL allowlist (T4).+ *+ * Both concurrent actors are the real routes mounted on one app over one stateful+ * MockGateway: the generic uninstall (`DELETE /admin/mcp-servers/:name`) and the+ * remote install saga (`POST /admin/mcp-servers/remote`, CIMD mode). The race is+ * forced from a gateway hook that runs one route inside the other's K8s call — no+ * allowlist write is hand-emulated (T1).+ *+ * The OAuth teardown's DB is a benign empty stub: these tests are about K8s order.+ */++vi.mock('node:dns/promises', () => ({+ lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]),+}))++vi.mock('../src/oauth/discovery.js', async importActual => {+ const actual = await importActual<typeof import('../src/oauth/discovery.js')>()+ return { ...actual, discoverRemoteOAuth: vi.fn() }+})++vi.mock('../src/db.js', async importActual => {+ const actual = await importActual<typeof import('../src/db.js')>()+ return { ...actual, pool: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) } }+})++const { config } = await import('../src/config.js')+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')+const { createAdminRemoteMcpRouter } = await import('../src/routes/admin/remoteMcp.js')++const NS = config.mcpServersNamespace++const PROBE_DNS = async () => ['93.184.216.34']+const ALIVE_TRANSPORT: PinnedTransport = async ({ method }) =>+ method === 'POST'+ ? { status: 401, headers: { 'www-authenticate': 'Bearer realm="OAuth"' }, bodyText: '' }+ : { status: 404, headers: {}, bodyText: '' }+const probe: typeof probeMcpTransport = (baseUrl, deps, opts) =>+ probeMcpTransport(baseUrl, { ...deps, transport: ALIVE_TRANSPORT, resolveDns: PROBE_DNS }, opts)++// Real Notion discovery with RFC 9207 advertised, derived from the real producer (T1)+// by the same documented substitution the remote-install route tests use.+let rfc9207Result: DiscoveryResult+beforeAll(async () => {+ const actual = await vi.importActual<typeof import('../src/oauth/discovery.js')>(+ '../src/oauth/discovery.js'+ )+ const as = JSON.parse(PILOTS.notion.as.json)+ as.authorization_response_iss_parameter_supported = true+ const pilot = { ...PILOTS.notion, as: { ...PILOTS.notion.as, json: JSON.stringify(as) } }+ const outcome = await actual.discoverRemoteOAuth(pilot.mcpUrl, {+ transport: makeDiscoveryTransport(pilot),+ resolveDns: PROBE_DNS,+ })+ if (!outcome.ok) throw new Error(`fixture discovery failed: ${outcome.error.kind}`)+ rfc9207Result = outcome.result+})++let prevContextsNs = ''+let prevCallbackBaseUrl = ''+beforeEach(() => {+ vi.mocked(discoverRemoteOAuth).mockResolvedValue({+ ok: true,+ result: rfc9207Result,+ } as DiscoveryOutcome)+ // The saga reads the Context in the gateway default namespace and the uninstall+ // in `contextsNamespace`; pinning them together makes both address one object, as+ // they do in production.+ prevContextsNs = config.contextsNamespace+ config.contextsNamespace = NS+ prevCallbackBaseUrl = config.oauthCallbackBaseUrl+ config.oauthCallbackBaseUrl = 'https://control.example.com'+})+afterEach(() => {+ config.contextsNamespace = prevContextsNs+ config.oauthCallbackBaseUrl = prevCallbackBaseUrl+ vi.restoreAllMocks()+})++function makeApp(gw: MockGateway) {+ const app = express()+ app.use(express.json())+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))+ app.use(createAdminRemoteMcpRouter(gw as unknown as K8sGateway, { probe }))+ return app+}++async function seed(gw: MockGateway, servers: string[]): Promise<void> {+ await gw.createResource(+ 'contexts',+ { metadata: { name: 'ctx-a' }, spec: { contextId: 'ctx-a', mcpServers: servers } },+ NS+ )+ for (const name of servers) {+ await gw.createResource('mcpservers', { metadata: { name }, spec: {} }, NS)+ }+}++const installBody = (serverName: string) => ({+ serverName,+ contextRef: 'ctx-a',+ baseUrl: 'https://mcp.notion.com/mcp',+ mode: 'cimd',+})++async function allowlist(gw: MockGateway): Promise<string[]> {+ const ctx = (await gw.getResource('contexts', 'ctx-a', NS)) as {+ spec: { mcpServers?: string[] }+ }+ return ctx.spec.mcpServers ?? []+}++describe('McpServer uninstall vs. Context allowlist writers', () => {+ // §7 row 7 (R3-H5b): a same-name reinstall can only create its CR once the old one+ // is gone. Stripping the allowlist AFTER that delete removed the reinstall's entry.+ it('a same-name reinstall that attaches right after the CR delete keeps its allowlist entry', async () => {+ const gw = new MockGateway(NS)+ await seed(gw, ['srv', 'other'])+ const app = makeApp(gw)++ let reinstallStatus: number | undefined+ let armed = true+ vi.spyOn(gw, 'deleteResource').mockImplementation(async (plural, name, ns, pre) => {+ const result = await MockGateway.prototype.deleteResource.call(gw, plural, name, ns, pre)+ if (armed && plural === 'mcpservers' && name === 'srv') {+ armed = false+ const reinstall = await request(app)+ .post('/admin/mcp-servers/remote')+ .send(installBody('srv'))+ reinstallStatus = reinstall.status+ }+ return result+ })++ const res = await request(app).delete('/admin/mcp-servers/srv')++ expect(res.status).toBe(200)+ expect(reinstallStatus).toBe(201)+ // The reinstalled server is live AND reachable through the Context.+ await expect(gw.getResource('mcpservers', 'srv', NS)).resolves.toBeTruthy()+ expect(await allowlist(gw)).toEqual(['other', 'srv'])+ })++ // §6.7 (R3-S6): an install of ANOTHER name read the Context before the uninstall's+ // strip and writes after it. Without a resourceVersion fence its write replays the+ // stale list and puts the uninstalled name back.+ it('an install of another name racing the strip does not reintroduce the uninstalled name', async () => {+ const gw = new MockGateway(NS)+ await seed(gw, ['old-srv'])+ const app = makeApp(gw)++ let uninstallStatus: number | undefined+ let armed = true+ vi.spyOn(gw, 'getResource').mockImplementation(async (plural, name, ns) => {+ const current = await MockGateway.prototype.getResource.call(gw, plural, name, ns)+ if (armed && plural === 'contexts' && name === 'ctx-a') {+ armed = false+ // The install's read is taken; the uninstall of old-srv then runs to+ // completion before the install gets to write.+ const stale = structuredClone(current)+ const uninstall = await request(app).delete('/admin/mcp-servers/old-srv')+ uninstallStatus = uninstall.status+ return stale+ }+ return current+ })++ const res = await request(app).post('/admin/mcp-servers/remote').send(installBody('new-srv'))++ expect(uninstallStatus).toBe(200)+ expect(res.status).toBe(201)+ await expect(gw.getResource('mcpservers', 'old-srv', NS)).rejects.toThrow()+ expect(await allowlist(gw)).toEqual(['new-srv'])+ })+})diff --git a/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts b/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.tsnew file mode 100644index 000000000..451a8c932--- /dev/null+++ b/control-api/test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts@@ -0,0 +1,164 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import express from 'express'+import { randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'+import type { DbClient } from '../src/db.js'+import type { K8sGateway } from '../src/k8s.js'+import { MockGateway } from './mockGateway.js'++/**+ * R3-H7 (§7 row 5, T1/T3/T4) — a failed grants purge on the generic McpServer+ * uninstall answers 503 repair_required with the CR still present, and repeating the+ * DELETE once the DB is healthy completes it: 200, CR gone, and the installation's+ * grant no longer readable.+ *+ * The grant is written by the REAL producer (`upsertOAuthGrant`, sealed with the CR+ * uid) and read back by the REAL fenced reader (`getOAuthGrant`) on a real Postgres.+ * The route's module-global `pool` is pointed at the test database; the only fault is+ * the purge statement throwing on the first request.+ *+ * CONTROL_API_REAL_PG_ADMIN_URL=postgres://<u>:<p>@<host>:5432/<db> npm test -- \+ * test/routes.mcpServerUninstall.repairRequired.realPostgres.integration.test.ts+ */++const dbHolder = vi.hoisted(() => ({+ target: null as null | { query: (text: string, values?: unknown[]) => Promise<unknown> },+ failGrantsPurge: false,+ // Evaluated per purge statement; lets a test fail only the post-delete purge.+ failGrantsPurgeWhen: null as null | (() => Promise<boolean>),+}))+vi.mock('../src/db.js', async importActual => {+ const actual = await importActual<typeof import('../src/db.js')>()+ return {+ ...actual,+ pool: {+ query: async (text: string, values?: unknown[]) => {+ if (/DELETE\s+FROM\s+oauth_grants/i.test(text)) {+ if (dbHolder.failGrantsPurge || (await dbHolder.failGrantsPurgeWhen?.())) {+ throw new Error('simulated oauth_grants purge failure')+ }+ }+ if (!dbHolder.target) throw new Error('test database not ready')+ return dbHolder.target.query(text, values)+ },+ },+ }+})++const { config } = await import('../src/config.js')+const { initDb } = await import('../src/db.js')+const { deriveOAuthEncryptionKey } = await import('../src/oauth/encryption.js')+const { getOAuthGrant, upsertOAuthGrant } = await import('../src/oauth/store.js')+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++const KEY = deriveOAuthEncryptionKey(config.oauthEncryptionKey)+const NS = config.mcpServersNamespace++describeRealPostgres('generic McpServer uninstall — repair_required (real Postgres)', () => {+ const database = `control_api_uninstall_${randomUUID().replace(/-/g, '')}`+ let adminPool: Pool+ let dbPool: Pool+ let db: DbClient++ beforeAll(async () => {+ if (!adminUrl) throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required')+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)+ dbPool = new Pool({ connectionString: databaseUrl(adminUrl, database) })+ await initDb({ connect: () => dbPool.connect() })+ db = { query: (text, values) => dbPool.query(text, values) }+ dbHolder.target = db+ })++ afterAll(async () => {+ dbHolder.target = null+ await dbPool?.end()+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)+ await adminPool.end()+ }+ })++ it('a failed purge keeps the CR (503); the repeated DELETE completes and no grant is visible', async () => {+ const name = `srv-rr-${randomUUID().slice(0, 8)}`+ const gw = new MockGateway(NS)+ const cr = (await gw.createResource('mcpservers', { metadata: { name }, spec: {} }, NS)) as {+ metadata: { uid: string }+ }+ const U = cr.metadata.uid+ const grantKey = {+ grantKind: 'user' as const,+ ownerKind: 'mcpserver' as const,+ recipeNamespace: NS,+ recipeName: name,+ userId: 'user-a',+ oauthClientId: 'client',+ }+ await upsertOAuthGrant(db, KEY, {+ ...grantKey,+ provider: 'google',+ accessToken: 'at-a',+ crUid: U,+ })+ const app = express()+ app.use(express.json())+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))++ dbHolder.failGrantsPurge = true+ let first: request.Response+ try {+ first = await request(app).delete(`/admin/mcp-servers/${name}`)+ } finally {+ dbHolder.failGrantsPurge = false+ }++ expect(first.status).toBe(503)+ expect(first.body).toMatchObject({+ error: 'mcp_server_uninstall_incomplete',+ outcome: 'repair_required',+ pending: ['oauth_grants'],+ })+ // Same installation, still there: the retry has the uid it needs.+ await expect(gw.getResource('mcpservers', name, NS)).resolves.toMatchObject({+ metadata: { uid: U },+ })++ // Only the post-delete purge (issued once the CR is gone) fails on the retry: the+ // grant must already be gone from the pre-delete teardown, and the post-delete+ // purge is hygiene that never changes the 200.+ const crGone = async () => {+ try {+ await gw.getResource('mcpservers', name, NS)+ return false+ } catch {+ return true+ }+ }+ dbHolder.failGrantsPurgeWhen = crGone+ let retry: request.Response+ try {+ retry = await request(app).delete(`/admin/mcp-servers/${name}`)+ } finally {+ dbHolder.failGrantsPurgeWhen = null+ }++ expect(retry.status).toBe(200)+ await expect(gw.getResource('mcpservers', name, NS)).rejects.toThrow()+ expect(await getOAuthGrant(db, KEY, { ...grantKey, crUid: U })).toBeNull()+ })+})diff --git a/control-api/test/routes.mcpServerUninstall.repairRequired.test.ts b/control-api/test/routes.mcpServerUninstall.repairRequired.test.tsnew file mode 100644index 000000000..0356f3584--- /dev/null+++ b/control-api/test/routes.mcpServerUninstall.repairRequired.test.ts@@ -0,0 +1,171 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import type { K8sGateway } from '../src/k8s.js'+import { MockGateway } from './mockGateway.js'++/**+ * Generic McpServer uninstall (`DELETE /admin/mcp-servers/:name`): every cleanup runs+ * before the CR delete, so a failed step answers 503 repair_required naming the step,+ * leaves the CR in place, and the same DELETE completes the job (R3-H7).+ *+ * Observed on a stateful MockGateway (T4: what survives, not which calls were made).+ * The OAuth teardown's DB is a benign empty stub; the grants path against a real+ * Postgres lives in routes.mcpServerUninstall.repairRequired.realPostgres.+ */++const dbMock = vi.hoisted(() => ({+ query: vi.fn(async (_text: string, _values?: unknown[]) => ({ rows: [], rowCount: 0 })),+}))+vi.mock('../src/db.js', async importActual => {+ const actual = await importActual<typeof import('../src/db.js')>()+ return { ...actual, pool: { query: dbMock.query } }+})++const { config } = await import('../src/config.js')+const { createAdminResourcesRouter } = await import('../src/routes/admin/resources.js')++const NS = config.mcpServersNamespace+const SERVER = 'srv'+const CREDENTIALS = `${SERVER}-credentials`++let prevContextsNs = ''+beforeEach(() => {+ prevContextsNs = config.contextsNamespace+ config.contextsNamespace = 'contexts-ns'+ dbMock.query.mockClear()+})+afterEach(() => {+ config.contextsNamespace = prevContextsNs+ vi.restoreAllMocks()+})++async function setup() {+ const gw = new MockGateway(NS)+ await gw.createResource('mcpservers', { metadata: { name: SERVER }, spec: {} }, NS)+ await gw.createResource(+ 'contexts',+ { metadata: { name: 'ctx-a' }, spec: { contextId: 'ctx-a', mcpServers: [SERVER, 'b'] } },+ 'contexts-ns'+ )+ gw.seedSecret(CREDENTIALS, NS, { stringData: { TOKEN: 'x' } })+ const app = express()+ app.use(express.json())+ app.use(createAdminResourcesRouter(gw as unknown as K8sGateway))+ return { gw, app }+}++async function allowlist(gw: MockGateway): Promise<string[]> {+ const ctx = (await gw.getResource('contexts', 'ctx-a', 'contexts-ns')) as {+ spec: { mcpServers?: string[] }+ }+ return ctx.spec.mcpServers ?? []+}++const grantsPurges = () =>+ dbMock.query.mock.calls.filter(([text]) => /DELETE\s+FROM\s+oauth_grants/i.test(text)).length++const conflict = () =>+ Object.assign(new Error('the object has been modified'), { statusCode: 409, code: 409 })+const serverError = () =>+ Object.assign(new Error('etcd unavailable'), { statusCode: 500, code: 500 })++describe('DELETE /admin/mcp-servers/:name — cleanup before delete, 503 repair_required', () => {+ it('a Context strip failure stops before any OAuth, Secret or CR change', async () => {+ const { gw, app } = await setup()+ vi.spyOn(gw, 'updateResource').mockRejectedValue(serverError())++ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(res.status).toBe(503)+ expect(res.body).toEqual({+ error: 'mcp_server_uninstall_incomplete',+ outcome: 'repair_required',+ pending: ['contexts'],+ deleted: [],+ })+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()+ await expect(gw.getSecret(CREDENTIALS, NS)).resolves.toBeTruthy()+ expect(grantsPurges()).toBe(0)+ })++ it('retries a Context 409 a bounded number of times, then reports contexts pending', async () => {+ const { gw, app } = await setup()+ const update = vi.spyOn(gw, 'updateResource').mockRejectedValue(conflict())++ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(res.status).toBe(503)+ expect(res.body.pending).toEqual(['contexts'])+ expect(update).toHaveBeenCalledTimes(3)+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()+ })++ it('absorbs a Context 409 from a concurrent status write and completes', async () => {+ const { gw, app } = await setup()+ const realUpdate = MockGateway.prototype.updateResource+ let first = true+ vi.spyOn(gw, 'updateResource').mockImplementation(async (plural, name, body, ns) => {+ if (first && plural === 'contexts') {+ first = false+ throw conflict()+ }+ return realUpdate.call(gw, plural, name, body, ns)+ })++ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(res.status).toBe(200)+ expect(await allowlist(gw)).toEqual(['b'])+ })++ it('a Secret delete failure keeps the CR and names secrets; the retry completes', async () => {+ const { gw, app } = await setup()+ const secretDelete = vi.spyOn(gw, 'deleteSecret').mockRejectedValueOnce(serverError())++ const first = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(first.status).toBe(503)+ expect(first.body.pending).toEqual(['secrets'])+ expect(first.body.deleted).toEqual(['Context/ctx-a (removed from allowlist)'])+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()+ await expect(gw.getSecret(CREDENTIALS, NS)).resolves.toBeTruthy()++ secretDelete.mockRestore()+ const retry = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(retry.status).toBe(200)+ await expect(gw.getResource('mcpservers', SERVER, NS)).rejects.toThrow()+ await expect(gw.getSecret(CREDENTIALS, NS)).rejects.toThrow()+ expect(await allowlist(gw)).toEqual(['b'])+ })++ it('an unreadable Secret stops the uninstall before any mutation', async () => {+ const { gw, app } = await setup()+ vi.spyOn(gw, 'getSecret').mockRejectedValueOnce(serverError())++ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(res.status).toBe(503)+ expect(res.body.pending).toEqual(['secrets'])+ expect(await allowlist(gw)).toEqual([SERVER, 'b'])+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()+ expect(grantsPurges()).toBe(0)+ })++ it('a CR delete failure reports mcp_server with every dependency already cleaned', async () => {+ const { gw, app } = await setup()+ vi.spyOn(gw, 'deleteResource').mockRejectedValueOnce(serverError())++ const res = await request(app).delete(`/admin/mcp-servers/${SERVER}`)++ expect(res.status).toBe(503)+ expect(res.body.pending).toEqual(['mcp_server'])+ expect(res.body.deleted).toEqual([+ 'Context/ctx-a (removed from allowlist)',+ `Secret/${CREDENTIALS}`,+ ])+ await expect(gw.getResource('mcpservers', SERVER, NS)).resolves.toBeTruthy()+ await expect(gw.getSecret(CREDENTIALS, NS)).rejects.toThrow()+ })+})diff --git a/control-api/test/routes.registryInstall.test.ts b/control-api/test/routes.registryInstall.test.tsindex c329b2971..8749e51d6 100644--- a/control-api/test/routes.registryInstall.test.ts+++ b/control-api/test/routes.registryInstall.test.ts@@ -1450,9 +1450,14 @@ describe('POST /admin/registry/install', () => {})})- it('preserves dependencies after an ambiguous concurrent Context change', async () => {+ // A concurrent Context writer moves the resourceVersion between the attach's read+ // and its write. The attach re-reads and re-applies its edit to the fresh list+ // instead of replaying the stale one (which would drop the other writer's change)+ // or failing the install.+ it('merges into a concurrently changed Context allowlist instead of replaying a stale one', async () => {vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)+ vi.mocked(reportInstall).mockResolvedValueOnce({ acknowledged: true, stored: true })const gw = new MockGateway('mcp-server')await gw.createResource('contexts', {@@ -1491,14 +1496,79 @@ describe('POST /admin/registry/install', () => {registryEntryVersion: '1.0.0',credentials: { AIRTABLE_API_KEY: ['context', 'race'].join('-') },})+ .expect(201)++ expect(res.body.contextUpdated).toBe(true)+ await expect(gw.getResource('contexts', 'default-context')).resolves.toMatchObject({+ spec: { mcpServers: ['concurrent-writer', 'my-airtable'] },+ })+ await expect(gw.getResource('mcpservers', 'my-airtable', 'mcp-server')).resolves.toMatchObject({+ metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },+ })+ await expect(gw.getSecret('my-airtable-credentials', 'mcp-server')).resolves.toMatchObject({+ metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },+ })+ })++ // Sibling of the merge above: the Context is rewritten before EVERY attach write, so+ // the bounded retry runs out. The install must not guess — the CR and its Secret stay+ // for repair and the other writers' entries are not clobbered.+ it('gives up after the bounded attach retries when the Context keeps changing', async () => {+ vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)+ vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)++ const gw = new MockGateway('mcp-server')+ await gw.createResource('contexts', {+ metadata: { name: 'default-context' },+ spec: { contextId: 'default-context', mcpServers: [] },+ })+ const originalUpdate = gw.updateResource.bind(gw)+ let attachWrites = 0+ vi.spyOn(gw, 'updateResource').mockImplementation(async (plural, name, body, ns) => {+ // Only the attach writes carry the new server; the ambiguous-outcome fence+ // rewrites the pre-attach snapshot and must reach the gateway untouched.+ const servers = (body.spec as { mcpServers?: string[] }).mcpServers ?? []+ if (plural === 'contexts' && servers.includes('my-airtable')) {+ attachWrites += 1+ const current = (await gw.getResource('contexts', name, ns)) as {+ metadata: { resourceVersion: string }+ spec: { mcpServers?: string[] }+ }+ await originalUpdate(+ 'contexts',+ name,+ {+ metadata: { resourceVersion: current.metadata.resourceVersion },+ spec: {+ ...current.spec,+ mcpServers: [...(current.spec.mcpServers ?? []), `writer-${attachWrites}`],+ },+ },+ ns+ )+ }+ return originalUpdate(plural, name, body, ns)+ })++ const app = makeApp(gw as unknown as import('../src/k8s.js').K8sGateway)+ const res = await request(app)+ .post('/admin/registry/install')+ .send({+ serverName: 'my-airtable',+ contextRef: 'default-context',+ registryEntryName: 'airtable-mcp',+ registryEntryVersion: '1.0.0',+ credentials: { AIRTABLE_API_KEY: ['context', 'storm'].join('-') },+ }).expect(503)expect(res.body).toMatchObject({error: 'registry_install_outcome_ambiguous',outcome: 'repair_required',})+ expect(attachWrites).toBe(3)await expect(gw.getResource('contexts', 'default-context')).resolves.toMatchObject({- spec: { mcpServers: ['concurrent-writer'] },+ spec: { mcpServers: ['writer-1', 'writer-2', 'writer-3'] },})await expect(gw.getResource('mcpservers', 'my-airtable', 'mcp-server')).resolves.toMatchObject({metadata: { uid: expect.any(String), resourceVersion: expect.any(String) },diff --git a/control-api/test/routes.resources.test.ts b/control-api/test/routes.resources.test.tsindex f832d3e67..086e6393b 100644--- a/control-api/test/routes.resources.test.ts+++ b/control-api/test/routes.resources.test.ts@@ -35,6 +35,9 @@ vi.mock('../src/db.js', async () => {advisoryLockModelName: async () => {},advisoryLockModelNames: async () => {},boundCarrierTransactionIdleTimeout: async () => {},+ // The McpServer uninstall runs its OAuth teardown before the CR delete and answers+ // 503 when it fails, so it needs a DB that answers (empty: nothing to tear down).+ pool: { query: async () => ({ rows: [], rowCount: 0 }) },}})@@ -1243,21 +1246,24 @@ describe('routes/resources', () => {config.contextsNamespace = 'contexts-ns'config.mcpServersNamespace = 'mcpservers-ns'- const gateway = {- // The uninstall reads the CR before deleting to capture metadata.uid (R3-H5).- getResource: vi- .fn()- .mockResolvedValue({ metadata: { name: 'mcp-a', uid: 'uid-mcp-a', resourceVersion: '1' } }),- deleteResource: vi.fn().mockResolvedValue({ deleted: true }),- deleteSecret: vi.fn().mockResolvedValue({ deleted: true }),- listResource: vi.fn().mockResolvedValue([- {- metadata: { name: 'ctx-a' },- spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-a', 'mcp-b'] },- },- ]),- updateResource: vi.fn().mockResolvedValue({}),- }+ // Stateful gateway: the Context and the CR carry the uid/resourceVersion the+ // apiserver always returns, which the fenced strip and delete depend on.+ const gateway = new MockGateway('mcpservers-ns')+ const cr = (await gateway.createResource(+ 'mcpservers',+ { metadata: { name: 'mcp-a' }, spec: {} },+ 'mcpservers-ns'+ )) as { metadata: { uid: string } }+ await gateway.createResource(+ 'contexts',+ {+ metadata: { name: 'ctx-a' },+ spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-a', 'mcp-b'] },+ },+ 'contexts-ns'+ )+ const deleteSpy = vi.spyOn(gateway, 'deleteResource')+ const updateSpy = vi.spyOn(gateway, 'updateResource')try {const app = express()@@ -1266,21 +1272,20 @@ describe('routes/resources', () => {await request(app).delete('/admin/mcp-servers/mcp-a').expect(200)- // The CR delete is now fenced on the uid read back before deletion (R3-H5).- expect(gateway.deleteResource).toHaveBeenCalledWith('mcpservers', 'mcp-a', 'mcpservers-ns', {- uid: 'uid-mcp-a',+ expect(deleteSpy).toHaveBeenCalledWith('mcpservers', 'mcp-a', 'mcpservers-ns', {+ uid: cr.metadata.uid,+ })+ await expect(gateway.getResource('mcpservers', 'mcp-a', 'mcpservers-ns')).rejects.toThrow()+ await expect(gateway.getResource('contexts', 'ctx-a', 'contexts-ns')).resolves.toMatchObject({+ spec: { contextId: 'ctx-a', description: 'desc', mcpServers: ['mcp-b'] },})- expect(gateway.listResource).toHaveBeenCalledWith('contexts', 'contexts-ns')- expect(gateway.updateResource).toHaveBeenCalledWith(+ // The strip is fenced on the Context identity it read.+ expect(updateSpy).toHaveBeenCalledWith('contexts','ctx-a',- {- spec: {- contextId: 'ctx-a',- description: 'desc',- mcpServers: ['mcp-b'],- },- },+ expect.objectContaining({+ metadata: { uid: expect.any(String), resourceVersion: '1' },+ }),'contexts-ns')} finally {