Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
refactor(control-api): route the registry uninstall through the shared orchestrator (R3-H4)
Diego Zacarias59cd7fb9fe42
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 33262cfd4..4e2244a6d 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.670",+ "version": "0.1.671","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.670",+ "version": "0.1.671","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex df4fc8e1f..727501af4 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.670",+ "version": "0.1.671","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/http/k8sError.ts b/control-api/src/http/k8sError.tsindex b8a642157..9d2329b35 100644--- a/control-api/src/http/k8sError.ts+++ b/control-api/src/http/k8sError.ts@@ -32,3 +32,29 @@ export function extractK8sError(err: unknown): { status: number; message: string}return null}++/**+ * Client-safe reason for a failed K8s call, for text that leaves the process (response+ * bodies, warnings). Unlike `extractK8sError` it never falls back to `err.message`: a+ * client-node `ApiException` builds that message from the full apiserver response,+ * response headers included. Only the metav1.Status `message` is used, then a fixed+ * status text; an error without a K8s status gets the caller's fixed fallback.+ */+export function k8sSafeFailureMessage(err: unknown, fallback: string): string {+ const status = extractK8sError(err)?.status+ if (status === undefined) return fallback+ const body = (err as { body?: unknown }).body+ let statusMessage: unknown+ if (body && typeof body === 'object') {+ statusMessage = (body as { message?: unknown }).message+ } else if (typeof body === 'string') {+ try {+ statusMessage = (JSON.parse(body) as { message?: unknown }).message+ } catch {+ // Not a metav1.Status: never forward the raw body.+ }+ }+ return typeof statusMessage === 'string' && statusMessage.length > 0+ ? statusMessage+ : `K8s error ${status}`+}diff --git a/control-api/src/oauth/mcpServerUninstall.ts b/control-api/src/oauth/mcpServerUninstall.tsindex 3fe438624..7306d121d 100644--- a/control-api/src/oauth/mcpServerUninstall.ts+++ b/control-api/src/oauth/mcpServerUninstall.ts@@ -1,5 +1,5 @@import type { DbClient } from '../db.js'-import { extractK8sError } from '../http/k8sError.js'+import { extractK8sError, k8sSafeFailureMessage } from '../http/k8sError.js'import type { K8sGateway } from '../k8s.js'import type { Logger } from '../observability/logger.js'import { mcpServerUninstallTeardownFailuresTotal } from '../observability/metrics.js'@@ -43,7 +43,19 @@ export type McpServerUninstallStage =export type McpServerUninstallResult =| { status: 'not_found' }| { status: 'completed'; crUid: string; deleted: string[]; crDeleteResponse: unknown }- | { status: 'incomplete'; pending: McpServerUninstallStage[]; deleted: string[] }+ | {+ status: 'incomplete'+ pending: McpServerUninstallStage[]+ deleted: string[]+ /** Per-resource reason, for callers that report it (registry `warnings`). */+ failures: UninstallFailure[]+ }++export interface UninstallFailure {+ /** `Kind/name`, same vocabulary as `deleted`. */+ resource: string+ message: string+}export type UninstallGateway = Pick<K8sGateway,@@ -102,9 +114,9 @@ export async function deleteCapturedSecrets(namespace: string,logger: Logger,awaitDeletion?: McpServerUninstallDeps['awaitDeletion']-): Promise<{ deleted: string[]; failed: boolean }> {+): Promise<{ deleted: string[]; failures: UninstallFailure[] }> {const deleted: string[] = []- let failed = false+ const failures: UninstallFailure[] = []for (const capture of captures) {if (capture.status !== 'ready') {if (capture.status !== 'absent') {@@ -125,7 +137,10 @@ export async function deleteCapturedSecrets(logger.info({ secretName: capture.name, namespace }, 'McpServer Secret already gone')continue}- failed = true+ failures.push({+ resource: `Secret/${capture.name}`,+ message: k8sSafeFailureMessage(err, 'unable to verify deletion'),+ })mcpServerUninstallTeardownFailuresTotal.inc({stage: capture.name.endsWith('-oauth-client') ? 'oauth_client_secret' : 'secrets',})@@ -135,7 +150,7 @@ export async function deleteCapturedSecrets()}}- return { deleted, failed }+ return { deleted, failures }}function readCrUid(cr: unknown): string | undefined {@@ -150,11 +165,10 @@ export async function uninstallMcpServer(const { gateway, logger } = depsconst { name, namespace } = targetconst deleted: string[] = []- const incomplete = (stage: McpServerUninstallStage[]): McpServerUninstallResult => ({- status: 'incomplete',- pending: stage,- deleted,- })+ const incomplete = (+ stage: McpServerUninstallStage[],+ failures: UninstallFailure[]+ ): McpServerUninstallResult => ({ status: 'incomplete', pending: stage, deleted, failures })// 1 · CR identity + Secret snapshots.let cr: unknown@@ -173,14 +187,27 @@ export async function uninstallMcpServer('McpServer read returned no metadata.uid; a real apiserver always sets it, so this ' +'points at a non-apiserver gateway or a broken proxy — retrying will not resolve it')- return incomplete(['mcp_server'])+ return incomplete(+ ['mcp_server'],+ [{ resource: `McpServer/${name}`, message: 'identity unavailable' }]+ )}const secretCaptures = await captureMcpServerSecrets(gateway, name, namespace, logger)- // An unreadable Secret now would be orphaned once the CR is gone; stop before any+ // A Secret that cannot be read, or read without the uid/resourceVersion its delete+ // must be fenced on, would be orphaned once the CR is gone. Stop before any// mutation so the retry re-captures it.- if (secretCaptures.some(c => c.status === 'read-failed')) {+ const unfenceable = secretCaptures.filter(+ c => c.status === 'read-failed' || c.status === 'identity-unavailable'+ )+ if (unfenceable.length > 0) {mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'secrets' })- return incomplete(['secrets'])+ return incomplete(+ ['secrets'],+ unfenceable.map(c => ({+ resource: `Secret/${c.name}`,+ message: c.status === 'read-failed' ? 'unable to verify identity' : 'identity unavailable',+ }))+ )}// 2 · Context allowlists.@@ -192,7 +219,15 @@ export async function uninstallMcpServer(} catch (err) {mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'contexts' })logger.error({ serverName: name, err }, 'Context allowlist cleanup failed on uninstall')- return incomplete(['contexts'])+ return incomplete(+ ['contexts'],+ [+ {+ resource: 'Context allowlists',+ message: k8sSafeFailureMessage(err, 'unable to update safely'),+ },+ ]+ )}// 3 · OAuth state of THIS installation.@@ -206,9 +241,16 @@ export async function uninstallMcpServer(if (teardown.dynamicClient === 'done') deleted.push(`DynamicClient/${name}`)if (teardown.grants === 'done') deleted.push(`OAuthGrants/${name}`)const oauthPending: McpServerUninstallStage[] = []- if (teardown.dynamicClient === 'failed') oauthPending.push('dynamic_client')- if (teardown.grants === 'failed') oauthPending.push('oauth_grants')- if (oauthPending.length > 0) return incomplete(oauthPending)+ const oauthFailures: UninstallFailure[] = []+ if (teardown.dynamicClient === 'failed') {+ oauthPending.push('dynamic_client')+ oauthFailures.push({ resource: `DynamicClient/${name}`, message: 'revocation failed' })+ }+ if (teardown.grants === 'failed') {+ oauthPending.push('oauth_grants')+ oauthFailures.push({ resource: `OAuthGrants/${name}`, message: 'purge failed' })+ }+ if (oauthPending.length > 0) return incomplete(oauthPending, oauthFailures)// 4 · Secrets, fenced on the step-1 snapshots.const secrets = await deleteCapturedSecrets(@@ -219,7 +261,7 @@ export async function uninstallMcpServer(deps.awaitDeletion)deleted.push(...secrets.deleted)- if (secrets.failed) return incomplete(['secrets'])+ if (secrets.failures.length > 0) return incomplete(['secrets'], secrets.failures)// 5 · The CR itself, fenced on the uid every cleanup above was bound to.let crDeleteResponse: unknown@@ -229,7 +271,15 @@ export async function uninstallMcpServer(} catch (err) {mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'mcp_server' })logger.error({ serverName: name, namespace, err }, 'McpServer delete failed on uninstall')- return incomplete(['mcp_server'])+ return incomplete(+ ['mcp_server'],+ [+ {+ resource: `McpServer/${name}`,+ message: k8sSafeFailureMessage(err, 'unable to verify deletion'),+ },+ ]+ )}deleted.push(`McpServer/${name}`)diff --git a/control-api/src/routes/admin/__tests__/mcpServerUninstallPurgesGrants.test.ts b/control-api/src/routes/admin/__tests__/mcpServerUninstallPurgesGrants.test.tsindex adf94e903..1f9655b76 100644--- a/control-api/src/routes/admin/__tests__/mcpServerUninstallPurgesGrants.test.ts+++ b/control-api/src/routes/admin/__tests__/mcpServerUninstallPurgesGrants.test.ts@@ -43,27 +43,26 @@ vi.mock('../../../db.js', async importActual => {// Import AFTER vi.mock so the swapped `pool` wires into the router.const { createAdminResourcesRouter } = await import('../resources.js')const { config } = await import('../../../config.js')+const { MockGateway } = await import('../../../../test/mockGateway.js')describe('DELETE /admin/mcp-servers/:name — purges oauth_grants on uninstall (H-3, R3-H5)', () => {const SERVER_NAME = 'gdrive'- const CR_UID = 'uid-gdrive-live'- function buildApp() {- const gateway = {- // The uninstall reads the CR before deleting to capture metadata.uid (R3-H5).- getResource: vi.fn(async () => ({- metadata: { name: SERVER_NAME, uid: CR_UID, resourceVersion: '7' },- })),- deleteResource: vi.fn(async () => ({ metadata: { name: SERVER_NAME } })),- deleteSecret: vi.fn(async () => ({})),- getSecret: vi.fn(async () => ({})),- listResource: vi.fn(async () => []),- updateResource: vi.fn(async () => ({})),- }+ // Stateful gateway (T1): the CR and any Secret carry the uid/resourceVersion the+ // apiserver always returns, which the fenced cleanup depends on.+ async function buildApp() {+ const gateway = new MockGateway(config.mcpServersNamespace)+ const cr = (await gateway.createResource(+ 'mcpservers',+ { metadata: { name: SERVER_NAME }, spec: {} },+ config.mcpServersNamespace+ )) as { metadata: { uid: string } }+ const deleteResource = vi.spyOn(gateway, 'deleteResource')+ const getResource = vi.spyOn(gateway, 'getResource')const app = express()app.use(express.json({ limit: '1mb' }))app.use('/api/v1', createAdminResourcesRouter(gateway as never))- return { app, gateway }+ return { app, crUid: cr.metadata.uid, deleteResource, getResource }}beforeEach(() => {@@ -72,7 +71,7 @@ describe('DELETE /admin/mcp-servers/:name — purges oauth_grants on uninstall (})it('executes a cr_uid-fenced server-scoped DELETE FROM oauth_grants during uninstall', async () => {- const { app } = buildApp()+ const { app, crUid } = await buildApp()const res = await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)expect(res.status).toBe(200)@@ -84,21 +83,17 @@ describe('DELETE /admin/mcp-servers/:name — purges oauth_grants on uninstall (// Before H-3 no such query is issued; before R3-H5 it carried no cr_uid fence.expect(purge).toBeDefined()expect(purge!.text).toContain('cr_uid = $3 OR cr_uid IS NULL')- expect(purge!.values).toEqual([config.mcpServersNamespace, SERVER_NAME, CR_UID])+ expect(purge!.values).toEqual([config.mcpServersNamespace, SERVER_NAME, crUid])})it('deletes the CR fenced on the uid read before delete', async () => {- const { app, gateway } = buildApp()+ const { app, crUid, deleteResource, getResource } = await buildApp()- await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)+ await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`).expect(200)- expect(gateway.getResource).toHaveBeenCalledWith(- 'mcpservers',- SERVER_NAME,- config.mcpServersNamespace- )+ expect(getResource).toHaveBeenCalledWith('mcpservers', SERVER_NAME, config.mcpServersNamespace)// deleteResource receives the uid precondition captured from the read CR.- const deleteArgs = gateway.deleteResource.mock.calls[0]- expect(deleteArgs[3]).toEqual({ uid: CR_UID })+ const deleteArgs = deleteResource.mock.calls[0]+ expect(deleteArgs[3]).toEqual({ uid: crUid })})})diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex bb636e37a..96c386c54 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -5,13 +5,19 @@ import { isIP } from 'node:net'import { parse as parseYaml } from 'yaml'import { z } from 'zod'import { config } from '../../config.js'+import { type DbClient, pool } from '../../db.js'import { asyncHandler } from '../../http/asyncHandler.js'-import { extractK8sError } from '../../http/k8sError.js'+import { extractK8sError, k8sSafeFailureMessage } from '../../http/k8sError.js'import { validateMcpServerSpecPreflight } from '../../http/validateMcpServerSpec.js'import { K8sGateway } from '../../k8s.js'import type { UiAuthedRequest } from '../../middleware/controlUIAuth.js'import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'+import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'import { GenericConfigSuggestionSchema } from '../../oauth/genericKnobs.js'+import {+ type McpServerUninstallResult,+ uninstallMcpServer,+} from '../../oauth/mcpServerUninstall.js'import { getOAuthProviderAdapter, isKnownOAuthProvider } from '../../oauth/providers.js'import { rootLogger } from '../../observability/logger.js'import { type AdminUserRecord, findAdminById } from '../../services/adminAuthService.js'@@ -65,11 +71,7 @@ import {invalidSecretTypeReason,} from '../../services/secretConstraints.js'import { findSecretReferenceState } from '../../services/secretReferenceService.js'-import {- SecretSnapshot,- isControlApiManagedSecret,- toSecretSnapshot,-} from '../../services/secretRepository.js'+import { SecretSnapshot, toSecretSnapshot } from '../../services/secretRepository.js'import {validateWorkflowRecipeEgressPreflight,validateWorkflowRecipeLimits,@@ -1730,7 +1732,21 @@ function logRegistryFenceFailure(input: {)}-export function createAdminRegistryRouter(gateway?: K8sGateway): Router {+export function createAdminRegistryRouter(+ gateway?: K8sGateway,+ deps: {+ /** DB for the McpServer uninstall's OAuth teardown; defaults to the shared pool. */+ uninstallDb?: DbClient+ } = {}+): Router {+ const uninstallDb: DbClient = deps.uninstallDb ?? {+ query: (text, values) => pool.query(text, values),+ }+ // Derived on first use: only the McpServer uninstall needs it, and deriving it here+ // would make every registry route depend on the OAuth key being configured.+ let oauthEncryptionKey: Buffer | undefined+ const uninstallEncryptionKey = () =>+ (oauthEncryptionKey ??= deriveOAuthEncryptionKey(config.oauthEncryptionKey))const router = Router()// GET /admin/registry/catalog — Catalog plus installed state for Control UI@@ -3992,92 +4008,42 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {return}} else {- // Uninstall MCP Server- let credentialSecretSnapshot: SecretSnapshot | null = null- const credentialSecretName = `${resourceName}-credentials`- let oauthClientSecretSnapshot: SecretSnapshot | null = null- const oauthClientSecretName = `${resourceName}-oauth-client`-- // Capture the credential identity before deleting the parent CR. A- // same-name Secret replacement can be created after the CR delete;- // reading by name afterwards would make the cleanup delete a- // different owner's object. The later delete is bound to this exact- // UID/RV snapshot.- try {- credentialSecretSnapshot = normalizeSecretSnapshot(- await gateway.getSecret(credentialSecretName, namespace),- credentialSecretName,- namespace- )- } catch (err) {- if (extractK8sError(err)?.status !== 404) {- res.status(503).json({- error: 'registry_uninstall_outcome_ambiguous',- outcome: 'repair_required',- resourceName,- resourceType,- namespace,- deleted,- warnings: [- ...warnings,- `Secret/${credentialSecretName}: ${err instanceof Error ? err.message : 'unable to verify identity'}`,- ],- })- return- }+ // Context allowlists hold bare server names and are stripped in the canonical+ // contexts namespace, so uninstalling a same-name McpServer from any other+ // namespace would unlist the canonical server from every Context.+ if (namespace !== config.mcpServersNamespace) {+ res.status(422).json({+ error: `McpServer resources always live in namespace "${config.mcpServersNamespace}"`,+ })+ return}-- // Same fence for the managed OAuth client Secret. It is created in the- // install saga WITHOUT ownerReferences (it predates the CR), so K8s GC- // never collects it and control-api must delete it here — but by-name- // after the CR delete is exposed to the same delete/recreate race a- // concurrent reinstall opens. Bind the later delete to this snapshot.- // Absent (404) is the norm for non-OAuth / reference-mode servers.+ const encryptionKey = uninstallEncryptionKey()+ let outcome: McpServerUninstallResulttry {- oauthClientSecretSnapshot = normalizeSecretSnapshot(- await gateway.getSecret(oauthClientSecretName, namespace),- oauthClientSecretName,- namespace+ outcome = await uninstallMcpServer(+ {+ gateway,+ db: uninstallDb,+ encryptionKey,+ dcrDeps: { logger: log },+ contextsNamespace: config.contextsNamespace,+ logger: log,+ awaitDeletion: ({ kind, name, namespace: ns }) =>+ kind === 'McpServer'+ ? waitForDeletion(+ () => gateway.getResource('mcpservers', name, ns),+ `McpServer/${name}`+ )+ : waitForDeletion(() => gateway.getSecret(name, ns), `Secret/${name}`),+ },+ { name: resourceName, namespace })} catch (err) {- if (extractK8sError(err)?.status !== 404) {- res.status(503).json({- error: 'registry_uninstall_outcome_ambiguous',- outcome: 'repair_required',- resourceName,- resourceType,- namespace,- deleted,- warnings: [- ...warnings,- `Secret/${oauthClientSecretName}: ${err instanceof Error ? err.message : 'unable to verify identity'}`,- ],- })- return- }- }-- try {- const current = await readResourceForRollback(gateway, 'mcpservers', {- metadata: { name: resourceName, namespace },- spec: {},- })- if (!current) {- warnings.push(`McpServer/${resourceName}: not found`)- } else {- await gateway.deleteResource(- 'mcpservers',- resourceName,- namespace,- resourcePreconditions(current)- )- await waitForDeletion(- () => gateway.getResource('mcpservers', resourceName, namespace),- `McpServer/${resourceName}`- )- deleted.push(`McpServer/${resourceName}`)- }- } catch (err) {+ // Only the CR read throws; nothing has been touched yet.+ log.error(+ { err, resourceName, namespace },+ 'McpServer read failed on registry uninstall'+ )res.status(503).json({error: 'registry_uninstall_outcome_ambiguous',outcome: 'repair_required',@@ -4087,153 +4053,40 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {deleted,warnings: [...warnings,- `McpServer/${resourceName}: ${err instanceof Error ? err.message : 'unable to verify deletion'}`,+ `McpServer/${resourceName}: ${k8sSafeFailureMessage(err, 'unable to read')}`,],})return}-- // Delete credential Secret- try {- if (credentialSecretSnapshot) {- await gateway.deleteSecret(- credentialSecretSnapshot.name,- credentialSecretSnapshot.namespace,- secretPreconditions(credentialSecretSnapshot)- )- await waitForDeletion(- () =>- gateway.getSecret(- credentialSecretSnapshot!.name,- credentialSecretSnapshot!.namespace- ),- `Secret/${credentialSecretSnapshot.name}`- )- deleted.push(`Secret/${credentialSecretSnapshot.name}`)- }- } catch (err) {- if (extractK8sError(err)?.status !== 404) {- res.status(503).json({- error: 'registry_uninstall_outcome_ambiguous',- outcome: 'repair_required',- resourceName,- resourceType,- namespace,- deleted,- warnings: [- ...warnings,- `Secret/${credentialSecretName}: ${err instanceof Error ? err.message : 'unable to verify deletion'}`,- ],- })- return- }- // Secret may not exist (no credentials)- }-- // Delete the managed OAuth client Secret (S1-U3, Step 3b). The name is- // DERIVED (`${serverName}-oauth-client`), but reference mode accepts- // arbitrary clientIdRef/clientSecretRef names, so an operator-owned- // Secret can carry this exact name by coincidence. UID/RV fencing proves- // same-object, not same-owner, so confirm control-api's install saga- // created it via the managed-by label before razing it; otherwise skip- // and warn. Fenced to the pre-CR-delete snapshot, a no-op when the- // server had no OAuth block or used reference mode with a different name.- try {- if (- oauthClientSecretSnapshot &&- !isControlApiManagedSecret(oauthClientSecretSnapshot)- ) {- log.warn(- {- secretName: oauthClientSecretSnapshot.name,- namespace: oauthClientSecretSnapshot.namespace,- },- 'Skipped OAuth client Secret cleanup: not managed by control-api'- )- } else if (oauthClientSecretSnapshot) {- await gateway.deleteSecret(- oauthClientSecretSnapshot.name,- oauthClientSecretSnapshot.namespace,- secretPreconditions(oauthClientSecretSnapshot)- )- await waitForDeletion(- () =>- gateway.getSecret(- oauthClientSecretSnapshot!.name,- oauthClientSecretSnapshot!.namespace- ),- `Secret/${oauthClientSecretSnapshot.name}`- )- deleted.push(`Secret/${oauthClientSecretSnapshot.name}`)- }- } catch (err) {- if (extractK8sError(err)?.status !== 404) {- res.status(503).json({- error: 'registry_uninstall_outcome_ambiguous',- outcome: 'repair_required',- resourceName,- resourceType,- namespace,- deleted,- warnings: [- ...warnings,- `Secret/${oauthClientSecretName}: ${err instanceof Error ? err.message : 'unable to verify deletion'}`,- ],- })- return- }- // Secret may not exist (no OAuth block, or reference mode)- }-- // Remove from Context allowlists- try {- const ctxList = (await gateway.listResource('contexts', namespace)) as Array<{- metadata?: { name?: string; uid?: string; resourceVersion?: string }- spec?: Record<string, unknown> & {- contextId?: string- description?: string- mcpServers?: string[]- }- }>- for (const ctx of ctxList) {- const name = ctx.metadata?.name- const servers = ctx.spec?.mcpServers ?? []- if (name && servers.includes(resourceName)) {- const uid = ctx.metadata?.uid- const resourceVersion = ctx.metadata?.resourceVersion- if (!uid || !resourceVersion) {- throw new Error(`Context/${name} identity unavailable; refusing stale update`)- }- await gateway.updateResource(- 'contexts',- name,- {- metadata: { uid, resourceVersion },- spec: {- ...ctx.spec,- contextId: ctx.spec?.contextId ?? name,- mcpServers: servers.filter(s => s !== resourceName),- } as Record<string, unknown>,- },- namespace- )- deleted.push(`Context/${name} (removed from allowlist)`)- }- }- } catch (err) {+ if (outcome.status === 'not_found') {+ // Every cleanup runs before the CR delete, so a missing CR is already+ // clean or was deleted out of band; cleaning by name would have no uid to+ // fence a same-name reinstall's state against.+ warnings.push(`McpServer/${resourceName}: not found`)+ } else if (outcome.status === 'incomplete') {res.status(503).json({- error: 'registry_uninstall_partial',+ // `ambiguous`: the CR delete (or its settle) could not be verified.+ // `partial`: a cleanup step failed and the CR was deliberately kept.+ // A CR read without a uid also reports `mcp_server`: that one is+ // deterministic (nothing was touched) but unreachable with a real+ // apiserver, so it shares the conservative code.+ error: outcome.pending.includes('mcp_server')+ ? 'registry_uninstall_outcome_ambiguous'+ : 'registry_uninstall_partial',outcome: 'repair_required',resourceName,resourceType,namespace,- deleted,- warnings: [- ...warnings,- `Context allowlists: ${err instanceof Error ? err.message : 'unable to update safely'}`,- ],+ pending: outcome.pending,+ deleted: [...deleted, ...outcome.deleted],+ warnings: [...warnings, ...outcome.failures.map(f => `${f.resource}: ${f.message}`)],})return+ } else if (outcome.status === 'completed') {+ deleted.push(...outcome.deleted)+ } else {+ const unhandled: never = outcome+ throw new Error(`unhandled uninstall outcome: ${JSON.stringify(unhandled)}`)}}diff --git a/control-api/test/routes.registryInstall.oauth.test.ts b/control-api/test/routes.registryInstall.oauth.test.tsindex b92ac6bfe..9e6e1d191 100644--- a/control-api/test/routes.registryInstall.oauth.test.ts+++ b/control-api/test/routes.registryInstall.oauth.test.ts@@ -52,10 +52,19 @@ afterEach(() => {config.oauthCallbackBaseUrl = savedCallbackBaseUrl})+// The McpServer uninstall tears OAuth state down before the CR delete; these cases+// hold no OAuth rows, and the real-Postgres path lives in+// routes.registryUninstall.realPostgres.+const emptyUninstallDb = { query: async () => ({ rows: [], rowCount: 0 }) }+function makeApp(gateway: MockGateway) {const app = express()app.use(express.json())- app.use(createAdminRegistryRouter(gateway as unknown as import('../src/k8s.js').K8sGateway))+ app.use(+ createAdminRegistryRouter(gateway as unknown as import('../src/k8s.js').K8sGateway, {+ uninstallDb: emptyUninstallDb,+ })+ )app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {res.status(500).json({ error: err instanceof Error ? err.message : 'unknown' })diff --git a/control-api/test/routes.registryInstall.test.ts b/control-api/test/routes.registryInstall.test.tsindex 8749e51d6..80521a759 100644--- a/control-api/test/routes.registryInstall.test.ts+++ b/control-api/test/routes.registryInstall.test.ts@@ -1,5 +1,6 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { ApiException } from '@kubernetes/client-node'import { lookup } from 'node:dns/promises'import request from 'supertest'import { config } from '../src/config.js'@@ -3354,6 +3355,10 @@ describe('SSRF remote URL validation', () => {// ── Uninstall flow (§9.5) ───────────────────────────────────────────────────describe('DELETE /admin/registry/uninstall/:serverName', () => {+ // The OAuth teardown runs before the CR delete; these cases have no OAuth state, and+ // the real-Postgres path is covered in routes.registryUninstall.realPostgres.+ const emptyUninstallDb = { query: async () => ({ rows: [], rowCount: 0 }) }+function makeApp(gw = new MockGateway('mcp-server')) {if (!(gw as unknown as { _seeded?: boolean })._seeded) {gw.createResource('mcpservers', {@@ -3368,7 +3373,11 @@ describe('DELETE /admin/registry/uninstall/:serverName', () => {}const app = express()app.use(express.json())- app.use(createAdminRegistryRouter(gw as unknown as import('../src/k8s.js').K8sGateway))+ app.use(+ createAdminRegistryRouter(gw as unknown as import('../src/k8s.js').K8sGateway, {+ uninstallDb: emptyUninstallDb,+ })+ )app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {res.status(500).json({ error: err instanceof Error ? err.message : 'unknown' })@@ -3471,15 +3480,37 @@ describe('DELETE /admin/registry/uninstall/:serverName', () => {expect(gw.secretReadChecks).toBeGreaterThanOrEqual(2)})- it('handles non-existent server gracefully', async () => {- const { app } = makeApp()+ // A missing CR answers 200 with a `not found` warning and touches NOTHING: every+ // cleanup runs before the CR delete, so a name-addressed cleanup here would have no+ // uid to fence against a same-name reinstall (whose Secret a pre-registered install+ // writes before its CR).+ it('handles a non-existent server as not found without cleaning up by name', async () => {+ const gw = new MockGateway('mcp-server')+ await gw.createResource('contexts', {+ metadata: { name: 'ctx1' },+ spec: { contextId: 'ctx1', mcpServers: ['nonexistent'] },+ })+ gw.seedSecret('nonexistent-credentials', 'mcp-server', { stringData: { K: 'v' } })+ ;(gw as unknown as { _seeded?: boolean })._seeded = true+ const { app } = makeApp(gw)const res = await request(app).delete('/admin/registry/uninstall/nonexistent').expect(200)- expect(res.body.resourceName).toBe('nonexistent')+ expect(res.body).toMatchObject({+ resourceName: 'nonexistent',+ deleted: [],+ warnings: ['McpServer/nonexistent: not found'],+ })+ await expect(gw.getSecret('nonexistent-credentials', 'mcp-server')).resolves.toBeTruthy()+ await expect(gw.getResource('contexts', 'ctx1')).resolves.toMatchObject({+ spec: { mcpServers: ['nonexistent'] },+ })})- it('does not delete a replacement Secret that wins after the resource delete', async () => {+ // The Secrets are now deleted BEFORE the CR, fenced on the snapshot taken with the+ // CR read. A replacement written in between is a different object: its delete is+ // refused (409), the uninstall stops with the CR kept, and the replacement survives.+ it('does not delete a replacement Secret written between the capture and its delete', async () => {const gw = new MockGateway('mcp-server')await gw.createResource('mcpservers',@@ -3493,32 +3524,141 @@ describe('DELETE /admin/registry/uninstall/:serverName', () => {});(gw as unknown as { _seeded?: boolean })._seeded = true- const originalDeleteResource = gw.deleteResource.bind(gw)+ const originalDeleteSecret = gw.deleteSecret.bind(gw)let raced = false- vi.spyOn(gw, 'deleteResource').mockImplementation(async (...args) => {- if (args[0] === 'mcpservers' && !raced) {+ vi.spyOn(gw, 'deleteSecret').mockImplementation(async (name, ns, precondition) => {+ if (name === secretName && !raced) {raced = true- const result = await originalDeleteResource(...args)- await gw.deleteSecret(secretName, 'mcp-server')+ await originalDeleteSecret(secretName, 'mcp-server')gw.seedSecret(secretName, 'mcp-server', {uid: 'uid-secret-replacement',resourceVersion: '1',})- return result}- return originalDeleteResource(...args)+ return originalDeleteSecret(name, ns, precondition)})const { app } = makeApp(gw)const res = await request(app).delete('/admin/registry/uninstall/secret-race').expect(503)expect(res.body).toMatchObject({- error: 'registry_uninstall_outcome_ambiguous',+ error: 'registry_uninstall_partial',outcome: 'repair_required',+ pending: ['secrets'],})await expect(gw.getSecret(secretName, 'mcp-server')).resolves.toMatchObject({metadata: { uid: 'uid-secret-replacement', resourceVersion: '1' },})+ await expect(gw.getResource('mcpservers', 'secret-race', 'mcp-server')).resolves.toBeTruthy()+ })++ // Contexts live in `config.contextsNamespace`; the strip used to list them in the+ // McpServer's namespace, missing every Context whenever the two differ.+ it('strips the allowlist in the contexts namespace, not the McpServer one', async () => {+ const prevContextsNs = config.contextsNamespace+ config.contextsNamespace = 'contexts-ns'+ try {+ const gw = new MockGateway('mcp-server')+ await gw.createResource(+ 'mcpservers',+ { metadata: { name: 'split-ns' }, spec: { image: 'test:1.0' } },+ 'mcp-server'+ )+ await gw.createResource(+ 'contexts',+ { metadata: { name: 'ctx1' }, spec: { contextId: 'ctx1', mcpServers: ['split-ns', 'b'] } },+ 'contexts-ns'+ )+ ;(gw as unknown as { _seeded?: boolean })._seeded = true+ const { app } = makeApp(gw)++ const res = await request(app).delete('/admin/registry/uninstall/split-ns').expect(200)++ expect(res.body.deleted).toContain('McpServer/split-ns')+ await expect(gw.getResource('contexts', 'ctx1', 'contexts-ns')).resolves.toMatchObject({+ spec: { mcpServers: ['b'] },+ })+ } finally {+ config.contextsNamespace = prevContextsNs+ }+ })++ // Allowlists hold bare names, so a same-name McpServer outside the canonical+ // namespace would take the canonical server's entries with it.+ it('refuses to uninstall an McpServer outside the canonical namespace, touching nothing', async () => {+ const gw = new MockGateway('mcp-server')+ await gw.createResource(+ 'mcpservers',+ { metadata: { name: 'installed-srv' }, spec: { image: 'test:homonym' } },+ 'sandbox-recipes'+ )+ const { app } = makeApp(gw)++ const res = await request(app)+ .delete('/admin/registry/uninstall/installed-srv?namespace=sandbox-recipes')+ .expect(422)++ expect(res.body.error).toContain(config.mcpServersNamespace)+ await expect(+ gw.getResource('mcpservers', 'installed-srv', 'sandbox-recipes')+ ).resolves.toBeTruthy()+ await expect(gw.getResource('contexts', 'ctx1')).resolves.toMatchObject({+ spec: { mcpServers: ['installed-srv'] },+ })+ })++ // A client-node ApiException's `.message` embeds the apiserver response headers;+ // nothing the uninstall reports may carry it.+ it('never reports a K8s exception message (response headers) in warnings', async () => {+ const leaky = () =>+ new ApiException(401, 'Unauthorized', undefined, { 'audit-id': 'leaked-audit-id' })+ const readGw = new MockGateway('mcp-server')+ ;(readGw as unknown as { _seeded?: boolean })._seeded = true+ vi.spyOn(readGw, 'getResource').mockRejectedValueOnce(leaky())+ const read = await request(makeApp(readGw).app)+ .delete('/admin/registry/uninstall/installed-srv')+ .expect(503)++ const { app, gw } = makeApp()+ vi.spyOn(gw, 'deleteResource').mockRejectedValueOnce(leaky())+ const del = await request(app).delete('/admin/registry/uninstall/installed-srv').expect(503)++ for (const body of [read.body, del.body]) {+ expect(body.warnings).toEqual(['McpServer/installed-srv: K8s error 401'])+ expect(JSON.stringify(body)).not.toMatch(/Headers:|leaked-audit-id/)+ }+ })++ // A Secret whose delete cannot be fenced (no uid/resourceVersion) would be orphaned+ // once the CR is gone, so the uninstall refuses to start instead of skipping it. A+ // real apiserver always sets both; the empty uid stands for a gateway that dropped it.+ it('stops before any change when a Secret has no identity to fence its delete on', async () => {+ const gw = new MockGateway('mcp-server')+ await gw.createResource('mcpservers', {+ metadata: { name: 'installed-srv' },+ spec: { image: 'test:1.0' },+ })+ await gw.createResource('contexts', {+ metadata: { name: 'ctx1' },+ spec: { contextId: 'ctx1', mcpServers: ['installed-srv'] },+ })+ gw.seedSecret('installed-srv-credentials', 'mcp-server', { uid: '', resourceVersion: '1' })+ ;(gw as unknown as { _seeded?: boolean })._seeded = true+ const { app } = makeApp(gw)++ const res = await request(app).delete('/admin/registry/uninstall/installed-srv').expect(503)++ expect(res.body).toMatchObject({+ error: 'registry_uninstall_partial',+ outcome: 'repair_required',+ pending: ['secrets'],+ deleted: [],+ warnings: ['Secret/installed-srv-credentials: identity unavailable'],+ })+ await expect(gw.getResource('mcpservers', 'installed-srv', 'mcp-server')).resolves.toBeTruthy()+ await expect(gw.getResource('contexts', 'ctx1')).resolves.toMatchObject({+ spec: { mcpServers: ['installed-srv'] },+ })})it('does not delete a same-name replacement that wins the uninstall race', async () => {diff --git a/control-api/test/routes.registryUninstall.realPostgres.integration.test.ts b/control-api/test/routes.registryUninstall.realPostgres.integration.test.tsnew file mode 100644index 000000000..03718ee53--- /dev/null+++ b/control-api/test/routes.registryUninstall.realPostgres.integration.test.ts@@ -0,0 +1,282 @@+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import { randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import request from 'supertest'+import type { DbClient } from '../src/db.js'+import type { K8sGateway } from '../src/k8s.js'+import { MockGateway } from './mockGateway.js'++/**+ * R3-H4 (§7 row 3, T1/T3/T4) — uninstalling an OAuth McpServer through the REGISTRY+ * route tears its grants down, so a same-name reinstall with the same+ * `oauth_client_id` starts with no user connected.+ *+ * Every layer is its real producer: the registry install creates the CR and the+ * managed client Secret; the state comes from `buildAuthorizeUrl`; the grant is+ * written by the real callback route (`GET /oauth-callback/:id`) sealed with the CR+ * uid. Only the provider's token endpoint is simulated (an external AS). Observed+ * through two real readers:+ * - `listUserGrantsForServer`, the store reader behind the admin per-server grants+ * listing, deliberately NOT fenced by the live uid (it must list rows to manage+ * them) — the residue a registry uninstall used to leave shows up here as a+ * still-connected user;+ * - the grant-existence sweep, fenced by the live uid.+ *+ * CONTROL_API_REAL_PG_ADMIN_URL=postgres://<u>:<p>@<host>:5432/<db> npm test -- \+ * test/routes.registryUninstall.realPostgres.integration.test.ts+ */++vi.mock('node:dns/promises', () => ({+ lookup: vi.fn(async () => [{ address: '93.184.216.34', family: 4 }]),+}))++vi.mock('../src/services/registryClient.js', () => ({+ searchEntries: vi.fn(),+ getEntry: vi.fn(),+ getEntryVersion: vi.fn(),+ getCredentialSchema: vi.fn(),+ getCategories: vi.fn(),+ reportInstall: vi.fn(),+ downloadBundle: vi.fn(),+ getDigest: vi.fn(),+ uploadArtifacts: vi.fn(),+ updateVersionMetadata: vi.fn(),+ deleteVersion: vi.fn(),+ publishEntry: vi.fn(),+ resolvePublishScope: vi.fn(),+ applyPublishScope: vi.fn((name: string | undefined) => name),+}))++const dbHolder = vi.hoisted(() => ({+ target: null as null | { query: (text: string, values?: unknown[]) => Promise<unknown> },+}))+vi.mock('../src/db.js', async importActual => {+ const actual = await importActual<typeof import('../src/db.js')>()+ return {+ ...actual,+ pool: {+ query: async (text: string, values?: unknown[]) => {+ if (!dbHolder.target) throw new Error('test database not ready')+ return dbHolder.target.query(text, values)+ },+ },+ }+})++const { config } = await import('../src/config.js')+const { initDb } = await import('../src/db.js')+const { buildAuthorizeUrl } = await import('../src/oauth/authorizeUrlHelper.js')+const { resolveServerOAuthSubject } = await import('../src/oauth/mcpServerOAuthSpec.js')+const { listUserGrantsForServer } = await import('../src/oauth/store.js')+const { createAdminRegistryRouter } = await import('../src/routes/admin/registry.js')+const { createOAuthCallbackRouter } = await import('../src/routes/external/oauthCallback.js')+const { resolveBatchGrantExistence } = await import('../src/routes/mcpOauth.js')+const { getCredentialSchema, getDigest, getEntryVersion, reportInstall } =+ await import('../src/services/registryClient.js')++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++const NS = 'mcp-server'+const SERVER = 'my-gmail'+const USER = 'user-9'++/** Baked-OAuth catalog entry, same shape as the registry OAuth install tests. */+const GMAIL_ENTRY = {+ id: '1',+ name: 'gmail-mcp',+ version: '1.0.0',+ entry_type: 'mcp-server',+ description: 'Gmail MCP server',+ author: 'clerum',+ origin: 'official',+ category: 'productivity',+ tags: ['gmail'],+ trust_level: 'high',+ quality_tier: 'verified',+ status: 'published',+ server_mode: 'local',+ transport: 'streamableHttp',+ recipe_type: null,+ mcp_server_meta: {+ imageRef: 'clerum/gmail-mcp:1.0.0',+ port: 3000,+ oauth: { provider: 'google', scopes: [] },+ },+ recipe_meta: null,+ artifact_refs: null,+ downloads: 0,+ installs: 0,+ created_at: '2026-03-01T00:00:00Z',+}++describeRealPostgres('registry uninstall tears OAuth grants down (real Postgres)', () => {+ const database = `control_api_registry_uninstall_${randomUUID().replace(/-/g, '')}`+ let adminPool: Pool+ let dbPool: Pool+ let db: DbClient+ let prevCallbackBase = ''+ let prevMcpNs = ''+ let prevContextsNs = ''++ beforeAll(async () => {+ if (!adminUrl) throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required')+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)+ dbPool = new Pool({ connectionString: databaseUrl(adminUrl, database) })+ await initDb({ connect: () => dbPool.connect() })+ db = { query: (text, values) => dbPool.query(text, values) }+ dbHolder.target = db+ })++ afterAll(async () => {+ dbHolder.target = null+ await dbPool?.end()+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)+ await adminPool.end()+ }+ })++ beforeEach(() => {+ vi.mocked(getDigest).mockResolvedValue({ digest: null })+ vi.mocked(getCredentialSchema).mockRejectedValue(new Error('No credential schema endpoint'))+ vi.mocked(reportInstall).mockResolvedValue({ acknowledged: true, stored: true })+ vi.mocked(getEntryVersion).mockResolvedValue(GMAIL_ENTRY as never)+ prevCallbackBase = config.oauthCallbackBaseUrl+ prevMcpNs = config.mcpServersNamespace+ prevContextsNs = config.contextsNamespace+ config.oauthCallbackBaseUrl = 'https://control.example.com'+ config.mcpServersNamespace = NS+ config.contextsNamespace = NS+ // The provider's token endpoint (an external AS) is the only simulated party.+ vi.stubGlobal(+ 'fetch',+ vi.fn(+ async () =>+ new Response(+ JSON.stringify({+ access_token: 'AT',+ refresh_token: 'RT',+ expires_in: 3600,+ token_type: 'Bearer',+ }),+ { status: 200, headers: { 'content-type': 'application/json' } }+ )+ )+ )+ })++ afterEach(() => {+ config.oauthCallbackBaseUrl = prevCallbackBase+ config.mcpServersNamespace = prevMcpNs+ config.contextsNamespace = prevContextsNs+ vi.unstubAllGlobals()+ })++ function makeApp(gw: MockGateway) {+ const app = express()+ app.use(express.json())+ app.use(createAdminRegistryRouter(gw as unknown as K8sGateway))+ app.use(createOAuthCallbackRouter(gw as unknown as K8sGateway))+ return app+ }++ const install = (app: express.Express) =>+ request(app)+ .post('/admin/registry/install')+ .send({+ serverName: SERVER,+ contextRef: 'default-context',+ registryEntryName: 'gmail-mcp',+ registryEntryVersion: '1.0.0',+ oauth: {+ scopes: ['https://www.googleapis.com/auth/gmail.readonly'],+ grantScope: 'user',+ secret: { mode: 'managed', clientId: 'cid-value', clientSecret: 'csec-value' },+ },+ })++ /** Mint the state with the real authorize-URL builder, then hit the real callback. */+ async function consent(app: express.Express, gw: MockGateway): Promise<request.Response> {+ const minted = await buildAuthorizeUrl(+ {+ subjectKind: 'mcp',+ mcpServerName: SERVER,+ oauthClientId: SERVER,+ userId: USER,+ grantKind: 'user',+ redirectUri: `https://control.example.com/api/v1/oauth-callback/${SERVER}`,+ },+ {+ recipeReader: { read: async () => null },+ mcpServerReader: {+ read: async name => {+ const resolved = resolveServerOAuthSubject(+ (await gw.getResource('mcpservers', name, NS)) as Parameters<+ typeof resolveServerOAuthSubject+ >[0]+ )+ return resolved ? { namespace: NS, ...resolved } : null+ },+ },+ secretReader: {+ read: async (name, ns) => {+ const raw = (await gw.getSecret(name, ns)) as { data?: Record<string, string> }+ return Object.fromEntries(+ Object.entries(raw.data ?? {}).map(([k, v]) => [+ k,+ Buffer.from(v, 'base64').toString('utf8'),+ ])+ )+ },+ },+ stateSecret: config.oauthStateHmacSecret,+ }+ )+ if (minted.kind !== 'ok') throw new Error(`authorize url not minted: ${minted.kind}`)+ const state = new URL(minted.authorizeUrl).searchParams.get('state')!+ return request(app).get(`/oauth-callback/${SERVER}`).query({ code: 'AUTH_CODE', state })+ }++ const sweep = (gw: MockGateway) =>+ resolveBatchGrantExistence(+ { db, gateway: gw as unknown as K8sGateway, mcpServersNamespace: NS },+ [{ mcpServerName: SERVER, userId: USER }]+ )++ it('a same-name reinstall starts with no connected user after a registry uninstall', async () => {+ const gw = new MockGateway(NS)+ await gw.createResource('contexts', {+ metadata: { name: 'default-context' },+ spec: { contextId: 'default-context', mcpServers: [] },+ })+ const app = makeApp(gw)++ expect((await install(app)).status).toBe(201)+ expect((await consent(app, gw)).status).toBe(200)+ // Precondition: the consent produced a connected user, visible to both readers.+ expect(await listUserGrantsForServer(db, { namespace: NS, name: SERVER })).toHaveLength(1)+ expect(await sweep(gw)).toEqual([{ mcpServerName: SERVER, userId: USER, exists: true }])++ const uninstall = await request(app).delete(`/admin/registry/uninstall/${SERVER}`)+ expect(uninstall.status).toBe(200)+ expect((await install(app)).status).toBe(201)++ // The reinstall has the same oauth_client_id; no user may appear connected to it.+ expect(await listUserGrantsForServer(db, { namespace: NS, name: SERVER })).toEqual([])+ expect(await sweep(gw)).toEqual([{ mcpServerName: SERVER, userId: USER, exists: false }])+ })+})