Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): close error-mapping and fencing gaps in Secret constraints
Alfredo Lopez9fe8f2ff7781
diff --git a/charts/clerum-crds/crds/mcpserver.yaml b/charts/clerum-crds/crds/mcpserver.yamlindex 894cad564..776aa5814 100644--- a/charts/clerum-crds/crds/mcpserver.yaml+++ b/charts/clerum-crds/crds/mcpserver.yaml@@ -19,6 +19,14 @@ metadata:{"path": "spec.imagePullSecrets","namespaceBinding": "resource"+ },+ {+ "path": "spec.oauth.clientIdRef",+ "namespaceBinding": "resource"+ },+ {+ "path": "spec.oauth.clientSecretRef",+ "namespaceBinding": "resource"}]}diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 7c8fd17f3..ad203e71e 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.547",+ "version": "0.1.548","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.547",+ "version": "0.1.548","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex afe89aa2b..1efcd543b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.547",+ "version": "0.1.548","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex ab00c0274..037ca3dfd 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -1081,8 +1081,17 @@ class RegistryInstallRollbackError extends Error {readonly code = 'registry_install_rollback_incomplete'readonly status = 500- constructor() {- super('registry install rollback could not be completed without risking another writer')+ /**+ * `subject` names the object left behind, e.g. `McpServer/foo`. A+ * compensation failure is the one outcome an operator must act on by hand,+ * and without the subject the 500 says only that something could not be+ * cleaned up. Kind and name only — never credential data.+ */+ constructor(subject?: string) {+ super(+ 'registry install rollback could not be completed without risking another writer' ++ (subject ? `: ${subject}` : '')+ )this.name = 'RegistryInstallRollbackError'}}@@ -1359,10 +1368,10 @@ async function waitForResourceDeletionOrReplacement(const current = await readResourceForRollback(gateway, plural, snapshot)const outcome = classifyCreatedResourceAfterDeleteFailure(snapshot, current)if (outcome === 'gone' || outcome === 'replaced') return outcome- if (outcome !== 'unchanged-original') throw new RegistryInstallRollbackError()+ if (outcome !== 'unchanged-original') throw new RegistryInstallRollbackError(label)await sleep(pollMs)}- throw new RegistryInstallRollbackError()+ throw new RegistryInstallRollbackError(label)}/** Delete only the CR created by this saga; never fall back to name-only delete. */@@ -1374,25 +1383,25 @@ async function rollbackCreatedResource(): Promise<void> {const name = (snapshot.metadata as { name?: string } | undefined)?.nameconst namespace = (snapshot.metadata as { namespace?: string } | undefined)?.namespace- if (!name || !namespace) throw new RegistryInstallRollbackError()+ if (!name || !namespace) throw new RegistryInstallRollbackError(label)for (let attempt = 1; attempt <= 2; attempt += 1) {try {await gateway.deleteResource(plural, name, namespace, resourcePreconditions(snapshot))const settled = await waitForResourceDeletionOrReplacement(gateway, plural, snapshot, label)- if (settled === 'replaced') throw new RegistryInstallRollbackError()+ if (settled === 'replaced') throw new RegistryInstallRollbackError(label)return} catch (err) {if (err instanceof RegistryInstallRollbackError) throw errconst current = await readResourceForRollback(gateway, plural, snapshot)const outcome = classifyCreatedResourceAfterDeleteFailure(snapshot, current)if (outcome === 'gone') return- if (outcome === 'replaced') throw new RegistryInstallRollbackError()+ if (outcome === 'replaced') throw new RegistryInstallRollbackError(label)if (outcome === 'unchanged-original' && attempt < 2) continue- throw new RegistryInstallRollbackError()+ throw new RegistryInstallRollbackError(label)}}- throw new RegistryInstallRollbackError()+ throw new RegistryInstallRollbackError(label)}/** Delete only the Secret object created by this saga; never fall back to name-only delete. */@@ -2172,9 +2181,18 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {// The Context allowlist is authoritative for host discovery. If this// step fails, the McpServer exists but is unreachable, so the install// must roll back rather than return a false success.+ // Contexts live in their own namespace, never in the McpServer's target+ // namespace. Naming it explicitly at every site keeps the read, the+ // write, the snapshot and the readback addressed to ONE object: while+ // the read and write relied on the gateway default and the snapshot and+ // readback used `targetNs`, any install with an explicit+ // `body.namespace` fenced against a different object than it wrote, so+ // a committed association could never classify as committed. The Host+ // association path below already passes `config.hostsNamespace` this way.+ const contextsNs = config.contextsNamespacelet contextBefore: RegistryResourceSnapshot | null = nulltry {- const ctx = (await gateway.getResource('contexts', contextRef)) as {+ const ctx = (await gateway.getResource('contexts', contextRef, contextsNs)) as {metadata?: { uid?: string; resourceVersion?: string }spec?: Record<string, unknown> & {contextId?: string@@ -2182,7 +2200,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {mcpServers?: string[]}}- contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, targetNs)+ contextBefore = normalizeRegistryResourceSnapshot(ctx, contextRef, contextsNs)if (!contextBefore.metadata?.uid || !contextBefore.metadata.resourceVersion) {throw Object.assign(new Error('Context identity is unavailable'), {statusCode: 503,@@ -2191,19 +2209,24 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {}const existing: string[] = ctx.spec?.mcpServers ?? []if (!existing.includes(serverName)) {- await gateway.updateResource('contexts', contextRef, {- metadata: {- uid: contextBefore.metadata.uid,- ...(ctx.metadata?.resourceVersion- ? { resourceVersion: ctx.metadata.resourceVersion }- : {}),+ await gateway.updateResource(+ 'contexts',+ contextRef,+ {+ metadata: {+ uid: contextBefore.metadata.uid,+ ...(ctx.metadata?.resourceVersion+ ? { resourceVersion: ctx.metadata.resourceVersion }+ : {}),+ },+ spec: {+ ...ctx.spec,+ contextId: ctx.spec?.contextId ?? contextRef,+ mcpServers: [...existing, serverName],+ } as Record<string, unknown>,},- spec: {- ...ctx.spec,- contextId: ctx.spec?.contextId ?? contextRef,- mcpServers: [...existing, serverName],- } as Record<string, unknown>,- })+ contextsNs+ )}} catch (err) {let associationOutcome: RegistryMutationOutcome = isDeterministicRegistryNoCommit(err)@@ -2214,7 +2237,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {gateway,'contexts',contextRef,- targetNs,+ contextsNs,contextBefore,spec =>Array.isArray(spec.mcpServers) &&diff --git a/control-api/src/routes/admin/resources.ts b/control-api/src/routes/admin/resources.tsindex f31582c12..8757580a5 100644--- a/control-api/src/routes/admin/resources.ts+++ b/control-api/src/routes/admin/resources.ts@@ -19,6 +19,10 @@ import {type MutableResourceSnapshot,} from '../../services/resourceService.js'import { secretKeyNames } from '../../services/secretKeyNames.js'+import {+ isRecipeOwnedSecret,+ secretIdentityPreconditions,+} from '../../services/secretRepository.js'import {ClerumResourceType,type ResourcePreconditions,@@ -64,12 +68,6 @@ function recordValue(value: unknown): Record<string, unknown> | null {: null}-function resourceVersionFromResource(resource: unknown): string | undefined {- const metadata = recordValue((resource as { metadata?: unknown } | null)?.metadata)- const resourceVersion = metadata?.resourceVersion- return typeof resourceVersion === 'string' && resourceVersion ? resourceVersion : undefined-}-function resourcePreconditionsFromResource(resource: unknown): ResourcePreconditions | undefined {const metadata = recordValue((resource as { metadata?: unknown } | null)?.metadata)const uid = metadata?.uid@@ -1092,42 +1090,121 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {}}+ // Capture the credential Secret identity BEFORE deleting the parent CR.+ // A same-name Secret can be created in the gap, and a name-addressed+ // delete afterwards would remove that replacement — a different owner's+ // object — on the strength of a decision made about something that no+ // longer exists. Every cleanup below is bound to the snapshot taken here.+ // Same reasoning, and same shape, as the registry uninstall path.+ const captureSecretForCleanup = async (+ secretName: string+ ): Promise<+ | { status: 'ready'; precondition: SecretPreconditions }+ | { status: 'absent' | 'recipe-owned' | 'identity-unavailable' | 'read-failed' }+ > => {+ let raw: unknown+ try {+ raw = await gateway.getSecret(secretName, ns)+ } catch (err) {+ if (extractK8sStatusCode(err) === 404) return { status: 'absent' }+ // Deleting the CR is the request's actual intent; the Secret is a+ // cascade. A failed read means we cannot fence the cascade delete, so+ // it is skipped and logged — the same policy this PR already applies+ // to the rollback paths ("rollback skipped: identity unavailable").+ // Failing the whole request here would leave the CR undeleted because+ // its dependent Secret could not be read.+ log.error(+ { secretName, namespace: ns, err },+ 'Secret cleanup capture failed; cascade delete will be skipped'+ )+ return { status: 'read-failed' }+ }+ // Recipe-owned Secrets belong to /admin/recipe-secrets. The mcp-secret+ // route refuses them with 409; deleting one here would route around+ // that guard.+ if (isRecipeOwnedSecret(raw)) return { status: 'recipe-owned' }+ const precondition = secretIdentityPreconditions(raw)+ if (!precondition) return { status: 'identity-unavailable' }+ return { status: 'ready', precondition }+ }++ const ccSecretCleanup =+ plural === 'communicationchannels' && ccSecretRefName+ ? await captureSecretForCleanup(ccSecretRefName)+ : null+ const mcpCredentialsSecretName = `${name}-credentials`+ const mcpSecretCleanup =+ plural === 'mcpservers' ? await captureSecretForCleanup(mcpCredentialsSecretName) : null+const deleted = await gateway.deleteResource(plural, name, ns)- if (plural === 'communicationchannels' && ccSecretRefName) {- try {- await gateway.deleteSecret(ccSecretRefName, ns)- log.info(- { secretName: ccSecretRefName, namespace: ns },- 'Deleted CommunicationChannel credentials Secret'+ if (plural === 'communicationchannels' && ccSecretRefName && ccSecretCleanup) {+ if (ccSecretCleanup.status !== 'ready') {+ log.warn(+ { secretName: ccSecretRefName, namespace: ns, reason: ccSecretCleanup.status },+ 'Skipped CommunicationChannel credentials Secret cleanup')- } catch (err) {- if (extractK8sStatusCode(err) === 404) {+ } else {+ try {+ await gateway.deleteSecret(ccSecretRefName, ns, ccSecretCleanup.precondition)log.info({ secretName: ccSecretRefName, namespace: ns },- 'CommunicationChannel credentials Secret already gone'- )- } else {- // CC is already gone; log and swallow so the operator gets a 200- // and can clean the orphan Secret manually if needed.- log.error(- { secretName: ccSecretRefName, namespace: ns, err },- 'CommunicationChannel delete succeeded but credentials cleanup failed'+ 'Deleted CommunicationChannel credentials Secret')+ } catch (err) {+ if (extractK8sStatusCode(err) === 404) {+ log.info(+ { secretName: ccSecretRefName, namespace: ns },+ 'CommunicationChannel credentials Secret already gone'+ )+ } else {+ // CC is already gone; log and swallow so the operator gets a 200+ // and can clean the orphan Secret manually if needed.+ log.error(+ { secretName: ccSecretRefName, namespace: ns, err },+ 'CommunicationChannel delete succeeded but credentials cleanup failed'+ )+ }}}}if (plural === 'mcpservers') {const contextsNs = resourceNamespace('contexts')- try {- await gateway.deleteSecret(`${name}-credentials`, ns)- log.info(- { secretName: `${name}-credentials`, namespace: ns },- 'Deleted MCP credentials Secret'+ if (mcpSecretCleanup && mcpSecretCleanup.status !== 'ready') {+ log.warn(+ {+ secretName: mcpCredentialsSecretName,+ namespace: ns,+ reason: mcpSecretCleanup.status,+ },+ 'Skipped MCP credentials Secret cleanup')- } catch (err) {- log.info({ serverName: name, err }, 'No MCP credentials Secret to delete')+ } else if (mcpSecretCleanup) {+ try {+ await gateway.deleteSecret(mcpCredentialsSecretName, ns, mcpSecretCleanup.precondition)+ log.info(+ { secretName: mcpCredentialsSecretName, namespace: ns },+ 'Deleted MCP credentials Secret'+ )+ } catch (err) {+ if (extractK8sStatusCode(err) === 404) {+ log.info(+ { secretName: mcpCredentialsSecretName, namespace: ns },+ 'MCP credentials Secret already gone'+ )+ } else {+ // The McpServer is already gone, so a 200 is still the honest+ // outcome — but this is a cleanup FAILURE, not an absence. The+ // previous catch reported every error class, 403 included, as+ // "no Secret to delete", telling the operator a credential had+ // been removed while it was still live.+ log.error(+ { secretName: mcpCredentialsSecretName, namespace: ns, err },+ 'McpServer delete succeeded but credentials cleanup failed'+ )+ }+ }}try {diff --git a/control-api/src/routes/admin/secrets.ts b/control-api/src/routes/admin/secrets.tsindex 6ead505d2..8c334fe07 100644--- a/control-api/src/routes/admin/secrets.ts+++ b/control-api/src/routes/admin/secrets.ts@@ -30,7 +30,11 @@ import {} from '../../services/mcpSecretRollbackPermitService.js'import { invalidSecretDataKeyReason } from '../../services/secretKeys.js'import { findSecretReferenceState } from '../../services/secretReferenceService.js'-import { secretIdentityPreconditions } from '../../services/secretRepository.js'+import {+ RECIPE_SECRET_LABEL_KEY,+ RECIPE_SECRET_LABEL_VALUE,+ secretIdentityPreconditions,+} from '../../services/secretRepository.js'import { toPublicDeleteSecretSummary, toPublicSecretSummary } from '../../services/secretService.js'import { SecretPreconditions, SecretUpsertRequest } from '../../types.js'import { listHostSecrets } from './hostSecrets.js'@@ -83,13 +87,6 @@ const BEDROCK_CREDENTIAL_KEYS: readonly string[] = PROVIDER_CREDENTIAL_SLOTS.bed)const VERTEX_SERVICE_ACCOUNT_KEY: string = PROVIDER_CREDENTIAL_SLOTS.vertex[0].dataKey-// The ownership label every WorkflowRecipe Secret carries. Declared at module-// scope so the recipe-secret routes below and the mcp-secret rotation guard-// read the SAME constant: a literal duplicated in either place would silently-// stop the guard from firing the day the other one changes.-const RECIPE_SECRET_LABEL_KEY = 'clerum.io/recipe-secret'-const RECIPE_SECRET_LABEL_VALUE = 'true'-// `evenfire-registry-pull` is control-api's own image-pull credential, self-provisioned// into every platform workload namespace (registryPullSecretService). The mcp-secret and// recipe-secret routes below write into that SAME set of namespaces, so the name is@@ -645,7 +642,14 @@ export function createAdminSecretsRouter(? { uid: identity.uid, resourceVersion: identity.resourceVersion }: nullif (!completeIdentity) {- res.status(503).json({ error: 'secret_identity_unavailable', outcome: 'repair_required' })+ // The Secret exists in the cluster; only its identity is missing. Echo+ // what IS known so the operator can name the object that needs repair —+ // the sibling permit-unavailable 503 below does the same.+ res.status(503).json({+ error: 'secret_identity_unavailable',+ outcome: 'repair_required',+ created: { name: name.trim(), namespace: targetNs },+ })return}const rollbackPermit: McpSecretRollbackPermit = {@@ -1181,11 +1185,28 @@ export function createAdminSecretsRouter(}}+ /**+ * Read a Secret for an identity-fenced mutation, mapping ONLY a 404 to+ * `null`. Every other read failure (403, 5xx, timeout) propagates: the+ * recipe-secret PUT/DELETE fence compares the live UID/RV against the+ * caller's, and collapsing an apiserver outage into "not found" would tell+ * the operator the Secret is gone while it still exists — the same false+ * proof `recipeExists` used to hand out before this PR made it fail closed.+ */+ async function readSecretOrNotFound(name: string, namespace: string): Promise<unknown | null> {+ try {+ return await gateway.getSecret(name, namespace)+ } catch (err) {+ if (extractHttpStatus(err) === 404) return null+ throw err+ }+ }+async function getRecipeSecret(name: string,namespace = config.sandboxNamespace): Promise<Record<string, unknown> | null> {- const existing = await gateway.getSecret(name, namespace).catch(() => null)+ const existing = await readSecretOrNotFound(name, namespace)if (!existing) return nullconst labels = ((existing as { metadata?: { labels?: Record<string, string> } }).metadata?.labels || {}) as Record<string, string>@@ -1369,9 +1390,7 @@ export function createAdminSecretsRouter(// Guardrail: refuse to mutate a workflow Secret that isn't a recipe// secret. Without this, the name alone could target coordinator tokens or// mcp-host/runtime auth Secrets that share an allowed namespace.- const existing = (await gateway- .getSecret(name.trim(), targetNamespace)- .catch(() => null)) as {+ const existing = (await readSecretOrNotFound(name.trim(), targetNamespace)) as {metadata?: {labels?: Record<string, string>uid?: stringdiff --git a/control-api/src/services/registryMutation.ts b/control-api/src/services/registryMutation.tsindex d19ca285d..f3290473b 100644--- a/control-api/src/services/registryMutation.ts+++ b/control-api/src/services/registryMutation.ts@@ -36,7 +36,11 @@ export function canonicalRegistryValue(value: unknown): unknown {return Object.fromEntries(Object.entries(value as Record<string, unknown>).filter(([, entry]) => entry !== undefined)- .sort(([left], [right]) => left.localeCompare(right))+ // Code-unit ordering, not localeCompare: this canonical form feeds a+ // SHA-256 digest that is persisted as an annotation, and collation+ // differs across ICU builds and default locales. A locale-dependent sort+ // would let two runtimes derive different digests for the same spec.+ .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)).map(([key, entry]) => [key, canonicalRegistryValue(entry)]))}diff --git a/control-api/src/services/secretConstraints.ts b/control-api/src/services/secretConstraints.tsindex d07eeb432..aec88d30c 100644--- a/control-api/src/services/secretConstraints.ts+++ b/control-api/src/services/secretConstraints.ts@@ -59,7 +59,15 @@ const REGISTRY_CREDENTIAL_WRITABLE_ANNOTATION_KEYS = [export type SecretWriteCapability = 'registryCredential' | 'registryPullSecret'-const CAPABILITY_ANNOTATION_KEYS: Record<SecretWriteCapability, readonly string[]> = {+/**+ * The EXACT platform annotation keys each capability grants, and nothing more.+ * Narrowness is the security property here: a capability exists so one internal+ * writer may set the specific keys it owns, never the `clerum.io/` prefix at+ * large. Exported so a test can pin the whole map — a per-key denylist can only+ * ever enumerate keys someone already thought of, and stays green when a new+ * one is added here.+ */+export const CAPABILITY_ANNOTATION_KEYS: Record<SecretWriteCapability, readonly string[]> = {registryCredential: REGISTRY_CREDENTIAL_WRITABLE_ANNOTATION_KEYS,registryPullSecret: ['clerum.io/pull-key-fingerprint'],}diff --git a/control-api/src/services/secretReferenceService.ts b/control-api/src/services/secretReferenceService.tsindex f5bdce767..9fddde2f5 100644--- a/control-api/src/services/secretReferenceService.ts+++ b/control-api/src/services/secretReferenceService.ts@@ -40,6 +40,16 @@ export type SecretReferenceCrdContract = {* Do not add a generic `*Ref` traversal here. Some CRDs expose ordinary* resource references with the same shape, and treating those as Secret* consumers would retain unrelated objects during compensation.+ *+ * Scanning boundary: only paths whose schema the CRD declares are covered.+ * Subtrees marked `x-kubernetes-preserve-unknown-fields: true` (Host+ * `spec.guardrails.rules[]` / `builtins[]`, LlmHook `spec.config`,+ * WorkflowRecipe step `config` / policy blobs) are opaque to this scanner and+ * to `test/crd.secretReferenceCoverage.test.ts`: a Secret name buried in one+ * of them is NOT a reference this module can see, so any future field that+ * consumes a Secret must be declared as a typed schema path AND listed here.+ * Anything left inside a preserve-unknown blob is by contract not a Secret+ * consumer.*/export const SECRET_REFERENCE_CRD_CONTRACTS = [{@@ -66,6 +76,13 @@ export const SECRET_REFERENCE_CRD_CONTRACTS = [{ path: 'spec.auth.secretRef', namespaceBinding: 'resource' },{ path: 'spec.envSecret', namespaceBinding: 'resource' },{ path: 'spec.imagePullSecrets', namespaceBinding: 'resource' },+ // spec.oauth.clientIdRef / clientSecretRef are required whenever+ // spec.oauth is set and each points at a Secret in the McpServer's own+ // namespace (mcpserver.yaml). They are read live by the OAuth broker+ // (oauth/authorizeUrlHelper.ts, oauth/tokenHelper.ts), so a Secret they+ // name is in use and must survive the delete and compensation guards.+ { path: 'spec.oauth.clientIdRef', namespaceBinding: 'resource' },+ { path: 'spec.oauth.clientSecretRef', namespaceBinding: 'resource' },],},{ plural: 'sharedfilesystems', fields: [] },diff --git a/control-api/src/services/secretRepository.ts b/control-api/src/services/secretRepository.tsindex bde2b5e8e..0ddb59e44 100644--- a/control-api/src/services/secretRepository.ts+++ b/control-api/src/services/secretRepository.ts@@ -1,5 +1,19 @@import type { SecretPreconditions } from '../types.js'+// The ownership label every WorkflowRecipe Secret carries. It lives here, next+// to the CAS helpers, so every route that can delete a Secret reads the SAME+// constant. A literal duplicated in a second module would silently stop that+// module's guard from firing the day this one changes.+export const RECIPE_SECRET_LABEL_KEY = 'clerum.io/recipe-secret'+export const RECIPE_SECRET_LABEL_VALUE = 'true'++/** True when the Secret is owned by a WorkflowRecipe and must not be deleted by other routes. */+export function isRecipeOwnedSecret(raw: unknown): boolean {+ const labels = (raw as { metadata?: { labels?: Record<string, string> } } | undefined)?.metadata+ ?.labels+ return labels?.[RECIPE_SECRET_LABEL_KEY] === RECIPE_SECRET_LABEL_VALUE+}+/** The subset of a Kubernetes Secret that callers are allowed to observe. */export interface SecretResource {metadata?: {diff --git a/control-api/src/services/secretService.ts b/control-api/src/services/secretService.tsindex 034916fa8..f01cd9f88 100644--- a/control-api/src/services/secretService.ts+++ b/control-api/src/services/secretService.ts@@ -23,6 +23,25 @@ export interface DeleteSecretSummary {deleted: true}+type Assert<T extends true> = T+type HasExactKeys<T, Keys extends PropertyKey> =+ Exclude<keyof T, Keys> extends never+ ? Exclude<Keys, keyof T> extends never+ ? true+ : false+ : false++// Compile-time ratchet: every admin Secret write route echoes one of these two+// DTOs, and the gateway now hands the routes a full `SecretSnapshot` (with+// `data`/`stringData`) that `toPublicSecretSummary` projects down. Adding a+// field here — `data`, `stringData`, `annotations`, anything — is the one-line+// change that would let Secret values or `last-applied-configuration` leave+// through an HTTP body, so it must fail the production typecheck, not a test.+type _SecretSummaryIsNamesOnly = Assert<HasExactKeys<SecretSummary, 'name' | 'namespace' | 'keys'>>+type _DeleteSecretSummaryIsNamesOnly = Assert<+ HasExactKeys<DeleteSecretSummary, 'name' | 'namespace' | 'deleted'>+>+export function toPublicSecretSummary(raw: SecretSummary | SecretSnapshot | Record<string, unknown> | null | undefined,fallbackName = '',@@ -416,9 +435,19 @@ export class SecretService {namespace?: string,precondition?: SecretPreconditions): Promise<DeleteSecretSummary> {+ const ns = namespace || this.defaultNamespace++ // Deleting is a mutation like any other, so the controller-owned type+ // guard applies here too. Without this read, delete was the one write path+ // that could destroy a Helm release ledger or a service-account token that+ // update/merge/removeKey all refuse to touch. The read is also the only+ // way to learn the type: a delete request carries no object.+ const existing = await this.coreApi.readNamespacedSecret({ namespace: ns, name })+ assertMutablePreservedSecretType(existing.type)+const hasPrecondition = Boolean(precondition?.uid || precondition?.resourceVersion)await this.coreApi.deleteNamespacedSecret({- namespace: namespace || this.defaultNamespace,+ namespace: ns,name,...(hasPrecondition && {body: {@@ -433,7 +462,7 @@ export class SecretService {})return toPublicDeleteSecretSummary({name,- namespace: namespace || this.defaultNamespace,+ namespace: ns,deleted: true,})}diff --git a/control-api/test/crd.secretReferenceCoverage.test.ts b/control-api/test/crd.secretReferenceCoverage.test.tsindex 43b9e81f7..119ac97a0 100644--- a/control-api/test/crd.secretReferenceCoverage.test.ts+++ b/control-api/test/crd.secretReferenceCoverage.test.ts@@ -34,6 +34,11 @@ const NON_SECRET_REFERENCE_PATHS = new Set(['communicationchannel.yaml:spec.hostRef','host.yaml:spec.contextRef','host.yaml:spec.llmPolicy.fallbacks[].credentialSlot',+ // Named Codex subscription key this Host is assigned to — a plain string+ // consumed by readHostCodexConnectionRef (hostSpecValidation.ts), not a+ // Secret reference. Contracting it would make the reference scanner retain+ // unrelated Secrets during compensation.+ 'host.yaml:spec.model.connectionRef','llmhook.yaml:spec.target.image.ref','mcpserver.yaml:spec.auth.secretKey','mcpserver.yaml:spec.contextRef',diff --git a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts b/control-api/test/db.oauthGrantsOwnerGeneralization.test.tsindex a61f1911c..87ccc5c10 100644--- a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts+++ b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts@@ -21,7 +21,13 @@ describe('0101 oauth_grants owner generalization migration', () => {const { CONTROL_API_MIGRATIONS } = await import('../src/db.js')const versions = CONTROL_API_MIGRATIONS.map(m => m.version)expect(versions).toContain('0106_oauth_grants_owner_generalization')- expect(versions.at(-1)).toBe('0108_llm_provider_attempts_sdk_link_on_delete_set_null')+ // Ordering, not tail position: this test guards where 0106 sits relative to+ // its neighbours. Pinning `.at(-1)` asserted that 0108 is the last+ // migration that will ever exist, which every subsequent additive+ // migration breaks — 0109_mcp_secret_rollback_permits did.+ expect(versions.indexOf('0108_llm_provider_attempts_sdk_link_on_delete_set_null')).toBeLessThan(+ versions.indexOf('0109_mcp_secret_rollback_permits')+ )expect(versions.indexOf('0100_seed_minimax_allowed_model')).toBeLessThan(versions.indexOf('0106_oauth_grants_owner_generalization'))diff --git a/control-api/test/routes.adminCommunicationChannelCredentials.test.ts b/control-api/test/routes.adminCommunicationChannelCredentials.test.tsindex 503c13707..c97074b83 100644--- a/control-api/test/routes.adminCommunicationChannelCredentials.test.ts+++ b/control-api/test/routes.adminCommunicationChannelCredentials.test.ts@@ -405,6 +405,19 @@ describe('admin communicationchannels — credentials cascade', () => {metadata: { name: 'foo', namespace: 'channels' },spec: { hostRef: 'h1', credentialsSecretRef: { name: 'cc-foo-credentials' } },})+ // The identity is captured BEFORE the CC is deleted, so the cascade delete+ // is fenced against the object that was actually inspected. Without this+ // read the route cannot fence, skips the delete, and every assertion below+ // about deleteSecret would be vacuous.+ gatewayMock.getSecret.mockResolvedValue({+ metadata: {+ name: 'cc-foo-credentials',+ namespace: 'channels',+ uid: 'uid-cc-foo-credentials',+ resourceVersion: '7',+ },+ data: {},+ })gatewayMock.deleteSecret.mockResolvedValue({})gatewayMock.deleteResource.mockResolvedValue({ deleted: true })@@ -416,7 +429,13 @@ describe('admin communicationchannels — credentials cascade', () => {'foo','channels')- expect(gatewayMock.deleteSecret).toHaveBeenCalledWith('cc-foo-credentials', 'channels')+ // A name-addressed delete would remove a same-name Secret created in the+ // gap after the CC was deleted. The preconditions bind the delete to the+ // object this request read.+ expect(gatewayMock.deleteSecret).toHaveBeenCalledWith('cc-foo-credentials', 'channels', {+ uid: 'uid-cc-foo-credentials',+ resourceVersion: '7',+ })// Order matters: CC delete must precede Secret delete so HCC's// SecretInformer doesn't observe the Secret-gone event while the CC is// still in the cache (which would produce a transient empty-data hash@@ -431,6 +450,19 @@ describe('admin communicationchannels — credentials cascade', () => {metadata: { name: 'foo', namespace: 'channels' },spec: { hostRef: 'h1', credentialsSecretRef: { name: 'gone' } },})+ // The Secret is present at capture time and disappears before the cascade+ // delete lands — that race is exactly what this test covers. Without the+ // read the route skips the delete entirely and the 200 below is reached+ // without ever exercising the 404 tolerance the test name claims.+ gatewayMock.getSecret.mockResolvedValue({+ metadata: {+ name: 'gone',+ namespace: 'channels',+ uid: 'uid-gone',+ resourceVersion: '3',+ },+ data: {},+ })const e404 = Object.assign(new Error('not found'), { statusCode: 404 })gatewayMock.deleteSecret.mockRejectedValue(e404)gatewayMock.deleteResource.mockResolvedValue({ deleted: true })@@ -443,6 +475,12 @@ describe('admin communicationchannels — credentials cascade', () => {'foo','channels')+ // Liveness witness: the 200 only means "tolerated" if the delete was+ // actually attempted and rejected. A skipped delete also yields 200.+ expect(gatewayMock.deleteSecret).toHaveBeenCalledWith('gone', 'channels', {+ uid: 'uid-gone',+ resourceVersion: '3',+ })})it('DELETE: non-404 Secret error after CC delete is logged but does not fail the request', async () => {@@ -450,6 +488,15 @@ describe('admin communicationchannels — credentials cascade', () => {metadata: { name: 'foo', namespace: 'channels' },spec: { hostRef: 'h1', credentialsSecretRef: { name: 'cc-foo-credentials' } },})+ gatewayMock.getSecret.mockResolvedValue({+ metadata: {+ name: 'cc-foo-credentials',+ namespace: 'channels',+ uid: 'uid-cc-foo-credentials',+ resourceVersion: '9',+ },+ data: {},+ })const e500 = Object.assign(new Error('boom'), { statusCode: 500 })gatewayMock.deleteSecret.mockRejectedValue(e500)gatewayMock.deleteResource.mockResolvedValue({ deleted: true })@@ -465,6 +512,12 @@ describe('admin communicationchannels — credentials cascade', () => {'foo','channels')+ // Liveness witness: a failed capture ALSO logs an error and returns 200, so+ // the spy alone cannot tell "delete failed" from "delete never attempted".+ expect(gatewayMock.deleteSecret).toHaveBeenCalledWith('cc-foo-credentials', 'channels', {+ uid: 'uid-cc-foo-credentials',+ resourceVersion: '9',+ })expect(errorSpy).toHaveBeenCalled()errorSpy.mockRestore()})diff --git a/control-api/test/routes.registryInstall.test.ts b/control-api/test/routes.registryInstall.test.tsindex 1758fea73..c329b2971 100644--- a/control-api/test/routes.registryInstall.test.ts+++ b/control-api/test/routes.registryInstall.test.ts@@ -609,6 +609,49 @@ describe('POST /admin/registry/install', () => {expect(getCredentialSchema).toHaveBeenCalledWith('airtable-mcp', '1.0.0')})+ // Contexts live in `config.contextsNamespace`, never in the McpServer's+ // target namespace. The Context read and write relied on the gateway default+ // while the snapshot and readback were addressed to `targetNs`, so an install+ // carrying an explicit `namespace` fenced against a different object than it+ // wrote and could never recover a committed association. The two defaults are+ // both 'mcp-server', which is why every other install test misses this.+ it('recovers a committed Context association when the install targets another namespace', async () => {+ vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)+ vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)+ vi.mocked(reportInstall).mockResolvedValueOnce({ acknowledged: true, stored: true })+ const { app, gw } = makeInstallApp()++ // The Context update commits and then loses its response, which is the only+ // path that exercises the readback.+ const updateSpy = vi.spyOn(gw, 'updateResource')+ updateSpy.mockImplementation(async (plural, name, body, ns) => {+ const result = await MockGateway.prototype.updateResource.call(gw, plural, name, body, ns)+ if (plural === 'contexts') {+ throw Object.assign(new Error('connection reset'), { statusCode: 500 })+ }+ return result+ })++ const res = await request(app)+ .post('/admin/registry/install')+ .send({+ serverName: 'ns-scoped',+ namespace: 'sandbox-recipes',+ contextRef: 'default-context',+ registryEntryName: 'airtable-mcp',+ registryEntryVersion: '1.0.0',+ credentials: { AIRTABLE_API_KEY: 'sk-test-token-123' },+ })+ .expect(201)++ expect(res.body.contextUpdated).toBe(true)+ // Witness that the write actually happened, and that it was addressed to+ // the contexts namespace rather than to the install target.+ const contextCall = updateSpy.mock.calls.find(call => call[0] === 'contexts')+ expect(contextCall).toBeDefined()+ expect(contextCall?.[3]).toBe('mcp-server')+ })+it('requires repair when a credential create response is lost after commit', async () => {vi.mocked(getEntryVersion).mockResolvedValueOnce(MOCK_ENTRY)vi.mocked(getCredentialSchema).mockResolvedValueOnce(MOCK_SCHEMA_REQUIRED)diff --git a/control-api/test/routes.secrets.test.ts b/control-api/test/routes.secrets.test.tsindex b602cdf81..a00350a0d 100644--- a/control-api/test/routes.secrets.test.ts+++ b/control-api/test/routes.secrets.test.ts@@ -143,7 +143,12 @@ describe('routes/secrets', () => {: []),getSecret: vi.fn(async (name: string) => {- if (!(name in labelledSecrets)) throw new Error('not found')+ // Mirror the real apiserver: a missing Secret is a 404-shaped+ // rejection. A bare Error here would only pass while the route+ // collapsed EVERY read failure into "not found".+ if (!(name in labelledSecrets)) {+ throw Object.assign(new Error('not found'), { statusCode: 404 })+ }const entry = labelledSecrets[name]return {metadata: {@@ -505,6 +510,65 @@ describe('routes/secrets', () => {expect(gateway.deleteSecret).not.toHaveBeenCalled()})+ // The 428 above only proves a body is REQUIRED; nothing proved the body is+ // COMPARED. Replacing the comparison with `false` — deleting whatever the+ // caller named, whoever replaced it in the meantime — left the whole suite+ // green. The witness in each case is the live read: it proves the route+ // reached the fence rather than bailing out earlier.+ it.each([+ { label: 'uid', body: { uid: 'uid-someone-elses', resourceVersion: '1' } },+ { label: 'resourceVersion', body: { uid: 'uid-r1', resourceVersion: '2' } },+ ])(+ 'refuses a recipe Secret DELETE whose $label no longer matches the live object',+ async testCase => {+ const gateway = createRecipeGateway()+ const app = express()+ app.use(express.json())+ app.use(createAdminSecretsRouter(gateway as never))++ const res = await request(app)+ .delete('/admin/recipe-secrets/r1')+ .send(testCase.body)+ .expect(409)++ expect(res.body).toMatchObject({+ error: 'secret_identity_changed',+ outcome: 'repair_required',+ })+ expect(gateway.getSecret).toHaveBeenCalled()+ expect(gateway.deleteSecret).not.toHaveBeenCalled()+ }+ )++ it('refuses a recipe Secret DELETE when the live object carries no identity', async () => {+ const gateway = createRecipeGateway()+ // A Secret that exists but whose identity the apiserver did not return:+ // deleting it would be a name-addressed delete with no fence at all.+ gateway.getSecret.mockResolvedValueOnce({+ metadata: {+ name: 'r1',+ namespace: 'sandbox-recipes',+ labels: { 'clerum.io/recipe-secret': 'true' },+ },+ data: {},+ } as never)+ const app = express()+ app.use(express.json())+ app.use(createAdminSecretsRouter(gateway as never))++ const res = await request(app)+ .delete('/admin/recipe-secrets/r1')+ .send({ uid: 'uid-r1', resourceVersion: '1' })+ .expect(503)++ expect(res.body).toMatchObject({+ error: 'secret_identity_unavailable',+ outcome: 'repair_required',+ })+ expect(gateway.getSecret).toHaveBeenCalled()+ expect(gateway.deleteSecret).not.toHaveBeenCalled()+ })+it('deletes a recipe secret from an allowed runtime namespace', async () => {const gateway = createRecipeGateway()gateway.getSecret.mockImplementation(async (name: string, namespace?: string) => {@@ -610,6 +674,75 @@ describe('routes/secrets', () => {await request(app).delete('/admin/recipe-secrets/ghost').expect(404)expect(gateway.deleteSecret).not.toHaveBeenCalled()})++ // Only a 404 may become "Recipe secret not found". Any other read failure+ // must propagate: the identity fence below cannot compare against a live+ // object it never saw, and a 404 here would tell the caller the Secret is+ // gone while it still exists on the cluster. The propagated error keeps+ // its own status: a 4xx from the apiserver is forwarded as that 4xx (the+ // Express default handler and `clerumErrorHandler` both read+ // `.statusCode`), while a 5xx or status-less failure surfaces as 500.+ it.each([+ {+ label: 'forbidden',+ error: Object.assign(new Error('forbidden'), { statusCode: 403 }),+ expectedStatus: 403,+ },+ {+ label: 'apiserver outage',+ error: Object.assign(new Error('boom'), { code: 500 }),+ expectedStatus: 500,+ },+ {+ label: 'status-less transport error',+ error: new Error('socket hang up'),+ expectedStatus: 500,+ },+ ])('DELETE fails loud when the live read fails with a $label', async testCase => {+ const gateway = createRecipeGateway()+ gateway.getSecret.mockRejectedValueOnce(testCase.error)+ const app = express()+ app.use(express.json())+ app.use(createAdminSecretsRouter(gateway as never))++ const res = await request(app)+ .delete('/admin/recipe-secrets/r1')+ .send({ uid: 'uid-r1', resourceVersion: '1' })+ .expect(testCase.expectedStatus)++ expect(res.body).not.toMatchObject({ error: 'Recipe secret not found' })+ // Liveness witness: the route reached the read and stopped there.+ expect(gateway.getSecret).toHaveBeenCalledOnce()+ expect(gateway.deleteSecret).not.toHaveBeenCalled()+ })++ it.each([+ {+ label: 'forbidden',+ error: Object.assign(new Error('forbidden'), { statusCode: 403 }),+ expectedStatus: 403,+ },+ {+ label: 'apiserver outage',+ error: Object.assign(new Error('boom'), { code: 500 }),+ expectedStatus: 500,+ },+ ])('PUT fails loud when the live read fails with a $label', async testCase => {+ const gateway = createRecipeGateway()+ gateway.getSecret.mockRejectedValueOnce(testCase.error)+ const app = express()+ app.use(express.json())+ app.use(createAdminSecretsRouter(gateway as never))++ const res = await request(app)+ .put('/admin/recipe-secrets')+ .send({ name: 'r1', data: { API_KEY: 'rotated' } })+ .expect(testCase.expectedStatus)++ expect(res.body).not.toMatchObject({ error: 'Recipe secret not found' })+ expect(gateway.getSecret).toHaveBeenCalledOnce()+ expect(gateway.updateSecret).not.toHaveBeenCalled()+ })})it('silently ignores ?namespace= query parameter and uses config namespace', async () => {diff --git a/control-api/test/routes.secretsNamesOnly.integration.test.ts b/control-api/test/routes.secretsNamesOnly.integration.test.tsindex 7e89770ac..1e7816e77 100644--- a/control-api/test/routes.secretsNamesOnly.integration.test.ts+++ b/control-api/test/routes.secretsNamesOnly.integration.test.ts@@ -135,9 +135,12 @@ describe('SECURITY (integration): admin secret-write responses are names-only',})it('DELETE /admin/secrets/:name returns a names-only summary, not a Secret body', async () => {- const { app } = makeApp({- written: fullSecret('svc-token', 'mcp-host', { token: b64(LEAK) }),- })+ // Delete reads the live object first to refuse controller-owned types, so+ // the Secret must exist. That read returns a FULL V1Secret with base64+ // `.data` — which is precisely what makes this a real leak test: the only+ // reason the response below is names-only is the SecretService trim.+ const live = fullSecret('svc-token', 'mcp-host', { token: b64(LEAK) })+ const { app } = makeApp({ existing: live, written: live })const res = await request(app).delete('/admin/secrets/svc-token').expect(200)expect(res.body).toMatchObject({ name: 'svc-token', deleted: true })expectNoSecretValues(res.body)diff --git a/control-api/test/services.mcpSecretRollbackPermit.test.ts b/control-api/test/services.mcpSecretRollbackPermit.test.tsindex acb4e37a2..d9d3d1c51 100644--- a/control-api/test/services.mcpSecretRollbackPermit.test.ts+++ b/control-api/test/services.mcpSecretRollbackPermit.test.ts@@ -125,6 +125,19 @@ describe('mcpSecretRollbackPermitService', () => {const sql = String(query.mock.calls[0]?.[0])expect(sql).toContain('SET claim_token = NULL')expect(sql).toContain('claim_token = $4')+ // The SQL text alone proves nothing about what $1..$4 carry: the release+ // must be bound to the hashed session, the exact resource, and the claim+ // token that was handed out, and the raw session JTI must never be a+ // parameter (it would land in pg logs and slow-query captures).+ expect(query.mock.calls[0]?.[1]).toEqual([+ expect.any(Buffer),+ 'mcp-server',+ 'linear-credentials',+ '11111111-1111-4111-8111-111111111111',+ ])+ expect((query.mock.calls[0]?.[1] as unknown[])[0]).toHaveLength(32)+ expect(query).toHaveBeenCalledOnce()+ expect(JSON.stringify(query.mock.calls)).not.toContain(permit.sessionJti)})it('finalizes only the matching claim after a terminal result', async () => {@@ -141,5 +154,17 @@ describe('mcpSecretRollbackPermitService', () => {const sql = String(query.mock.calls[0]?.[0])expect(sql).toContain('DELETE FROM mcp_secret_rollback_permits')expect(sql).toContain('claim_token = $4')+ // Same binding proof as release: a finalize that matched on anything+ // broader than (session hash, namespace, name, claim token) could delete a+ // permit another admin session still needs for its own rollback.+ expect(query.mock.calls[0]?.[1]).toEqual([+ expect.any(Buffer),+ 'mcp-server',+ 'linear-credentials',+ '11111111-1111-4111-8111-111111111111',+ ])+ expect((query.mock.calls[0]?.[1] as unknown[])[0]).toHaveLength(32)+ expect(query).toHaveBeenCalledOnce()+ expect(JSON.stringify(query.mock.calls)).not.toContain(permit.sessionJti)})})diff --git a/control-api/test/services.secretReferenceService.test.ts b/control-api/test/services.secretReferenceService.test.tsindex cf45cecc2..f34b64502 100644--- a/control-api/test/services.secretReferenceService.test.ts+++ b/control-api/test/services.secretReferenceService.test.ts@@ -28,6 +28,24 @@ describe('findSecretReferenceState', () => {resourceNamespace: config.mcpServersNamespace,resource: { spec: { imagePullSecrets: [{ name: referencedName }] } },},+ {+ label: 'McpServer OAuth client ID secret',+ namespace: config.mcpServersNamespace,+ plural: 'mcpservers',+ resourceNamespace: config.mcpServersNamespace,+ resource: {+ spec: { oauth: { clientIdRef: { name: referencedName, key: 'client_id' } } },+ },+ },+ {+ label: 'McpServer OAuth client secret',+ namespace: config.mcpServersNamespace,+ plural: 'mcpservers',+ resourceNamespace: config.mcpServersNamespace,+ resource: {+ spec: { oauth: { clientSecretRef: { name: referencedName, key: 'client_secret' } } },+ },+ },{label: 'LlmHook image pull secret',namespace: config.llmHooksNamespace,diff --git a/control-api/test/services.secretService.test.ts b/control-api/test/services.secretService.test.tsindex 9eb7d0335..8fb356380 100644--- a/control-api/test/services.secretService.test.ts+++ b/control-api/test/services.secretService.test.ts@@ -3,9 +3,11 @@ import * as k8s from '@kubernetes/client-node'import { extractHttpStatus } from '../src/k8s.js'import {ALLOWED_SECRET_TYPES,+ CAPABILITY_ANNOTATION_KEYS,DangerousAnnotationError,InvalidSecretTypeError,REGISTRY_SECRET_PRESERVED_ANNOTATION_KEYS,+ type SecretWriteCapability,dangerousAnnotationKeyReason,invalidSecretTypeReason,resolveSecretAnnotationsForReplace,@@ -914,6 +916,34 @@ describe('SecretService — preserved Secret state is constrained before mutatioexpect(testCase.write()).not.toHaveBeenCalled()})+ // Delete is a mutation like any other, and it was the one write path with no+ // controller-owned-type guard: update/merge/removeKey all refuse a Helm+ // release ledger or a service-account token, while delete destroyed it. A+ // delete request carries no object, so the guard's read is also the only way+ // to learn the type — which makes that read the liveness witness for the+ // negative assertion below.+ it.each(NON_MUTABLE_PRESERVED_SECRET_TYPES)(+ 'refuses to delete a controller-owned %s Secret',+ async type => {+ coreApi.readNamespacedSecret.mockResolvedValueOnce({ metadata: {}, type })++ await expect(svc.deleteSecret('s', 'test-ns')).rejects.toBeInstanceOf(InvalidSecretTypeError)+ expect(coreApi.readNamespacedSecret).toHaveBeenCalledOnce()+ expect(coreApi.deleteNamespacedSecret).not.toHaveBeenCalled()+ }+ )++ it.each(MUTABLE_PRESERVED_SECRET_TYPES)('allows deleting a non-managed %s Secret', async type => {+ coreApi.readNamespacedSecret.mockResolvedValueOnce({ metadata: {}, type })++ await expect(svc.deleteSecret('s', 'test-ns')).resolves.toMatchObject({+ name: 's',+ namespace: 'test-ns',+ deleted: true,+ })+ expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledOnce()+ })+it.each(MUTABLE_PRESERVED_SECRET_TYPES)('allows update when an omitted type preserves the non-managed %s type',async type => {@@ -1249,6 +1279,58 @@ describe('SecretService — platform annotations (clerum.io/) blocked by defaultexpect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()})+ // The security property of a capability is its NARROWNESS: it grants one+ // internal writer the exact keys it owns, never the `clerum.io/` prefix at+ // large. Every other test here enumerates keys that must be REFUSED, and a+ // denylist can only name keys someone already thought of — adding+ // 'clerum.io/anything' to registryCredential left the whole suite green.+ // These two tests pin the grant side instead: the first makes any widening+ // RED until it is reviewed on this line, the second proves the pinned map is+ // the one the guard actually reads.+ describe('capability allowlists are exhaustively pinned', () => {+ const EXPECTED_CAPABILITY_KEYS: Record<SecretWriteCapability, readonly string[]> = {+ registryCredential: [+ 'clerum.io/catalog-id',+ 'clerum.io/catalog-version',+ 'clerum.io/trust-level',+ 'clerum.io/registry-operation-id',+ ],+ registryPullSecret: ['clerum.io/pull-key-fingerprint'],+ }++ it('grants exactly the reviewed keys, and no capability grants any other', () => {+ expect(Object.keys(CAPABILITY_ANNOTATION_KEYS).sort()).toEqual(+ Object.keys(EXPECTED_CAPABILITY_KEYS).sort()+ )+ for (const [capability, expected] of Object.entries(EXPECTED_CAPABILITY_KEYS)) {+ const actual = CAPABILITY_ANNOTATION_KEYS[capability as SecretWriteCapability]+ expect([...actual].sort()).toEqual([...expected].sort())+ }+ })++ it('the pinned map is the set the guard enforces, and each capability is isolated', () => {+ for (const capability of Object.keys(EXPECTED_CAPABILITY_KEYS) as SecretWriteCapability[]) {+ const granted = EXPECTED_CAPABILITY_KEYS[capability]+ // Liveness witness: the guard must actually let these through, or the+ // refusals below would pass simply because everything is refused.+ for (const key of granted) {+ expect(dangerousAnnotationKeyReason(key, { capability })).toBeNull()+ }+ // An unlisted sibling under the same platform prefix stays refused.+ // This is the assertion the single-key denylist could not make.+ expect(dangerousAnnotationKeyReason('clerum.io/anything', { capability })).not.toBeNull()+ // Holding one capability must not grant another capability's keys.+ for (const [other, otherKeys] of Object.entries(EXPECTED_CAPABILITY_KEYS)) {+ if (other === capability) continue+ for (const key of otherKeys) {+ if (granted.includes(key)) continue+ expect(dangerousAnnotationKeyReason(key, { capability })).not.toBeNull()+ }+ }+ }+ })+ })+it('rejects a mixed payload containing both safe and platform keys (without opt-out)', async () => {const err = await svc.createSecret({diff --git a/control-ui/components/CreateMcpServerForm/index.tsx b/control-ui/components/CreateMcpServerForm/index.tsxindex 130ba6baa..6f7bb3280 100644--- a/control-ui/components/CreateMcpServerForm/index.tsx+++ b/control-ui/components/CreateMcpServerForm/index.tsx@@ -696,9 +696,11 @@ export function CreateMcpServerForm({}}const primaryError = formatCreateError(submitError)+ // Name the Secret: "review the Secret" is only actionable when the+ // operator knows which object was left behind.setError(cleanupError- ? `${primaryError} Cleanup of the created Secret also failed: ${formatCreateError(+ ? `${primaryError} Cleanup of the created Secret "${cleanupSecretName}" also failed: ${formatCreateError(cleanupError)}. Refresh the page and review the Secret before taking further action.`: primaryErrordiff --git a/control-ui/components/__tests__/CreateMcpServerForm.test.tsx b/control-ui/components/__tests__/CreateMcpServerForm.test.tsxindex 5caf974a2..d789d7ec0 100644--- a/control-ui/components/__tests__/CreateMcpServerForm.test.tsx+++ b/control-ui/components/__tests__/CreateMcpServerForm.test.tsx@@ -445,7 +445,13 @@ describe('CreateMcpServerForm — submit', () => {await screen.findByText(/shared OAuth identity.*original access scope/i)).toBeInTheDocument()expect(api.deleteMcpServer).toHaveBeenCalledWith('shared-drive')- expect(api.deleteMcpSecret).toHaveBeenCalledWith('shared-drive-oauth')+ // The rollback delete is compare-and-swap fenced: it must carry the+ // identity createMcpSecret returned, so it can never delete a Secret some+ // other writer replaced in between.+ expect(api.deleteMcpSecret).toHaveBeenCalledWith('shared-drive-oauth', {+ uid: 'uid-test-credentials',+ resourceVersion: '1',+ })expect(api.getContext).not.toHaveBeenCalled()expect(api.updateContext).not.toHaveBeenCalled()})@@ -867,7 +873,7 @@ describe('CreateMcpServerForm — envSecret guardrails', () => {expect(screen.getByText(/Connector create failed/)).toBeInTheDocument()expect(screen.getByText(- /Cleanup of the created Secret also failed: 428 Precondition Required - current identity required\. Refresh the page and review the Secret before taking further action\./+ /Cleanup of the created Secret "mixed-version-credentials" also failed: 428 Precondition Required - current identity required\. Refresh the page and review the Secret before taking further action\./)).toBeInTheDocument()})diff --git a/control-ui/e2e/qa-recorder-connector-edit.spec.ts b/control-ui/e2e/qa-recorder-connector-edit.spec.tsindex c43aecb16..28f9e31d3 100644--- a/control-ui/e2e/qa-recorder-connector-edit.spec.ts+++ b/control-ui/e2e/qa-recorder-connector-edit.spec.ts@@ -248,6 +248,13 @@ test.describe('optional QA recorder: Control UI connector edit', () => {const secretKey = 'api-key'const envVar = 'QA_RECORDER_API_KEY'let cleanupIdentity: SecretIdentity | null = null+ // Cleanup deletes only what this run actually created. Each flag flips+ // right after the create succeeded, so a body that fails before a step+ // never triggers a delete (and a spurious 404 assertion) for that object.+ let secretCreated = false+ let contextCreated = false+ let connectorCreated = false+ let primaryFailure: unknown = nulltry {await loginThroughUi(page, credentials)@@ -264,6 +271,7 @@ test.describe('optional QA recorder: Control UI connector edit', () => {data: { [secretKey]: 'initial-value-123' },})expect(secretRes.status, `create Secret: ${JSON.stringify(secretRes.data)}`).toBeLessThan(300)+ secretCreated = truecleanupIdentity = requireSecretIdentity(secretRes.data, 'stage cleanup identity')@@ -276,6 +284,7 @@ test.describe('optional QA recorder: Control UI connector edit', () => {},})expect(ctxRes.status, `create context: ${JSON.stringify(ctxRes.data)}`).toBeLessThan(300)+ contextCreated = trueconst srvRes = await api(page.request, 'POST', '/api/v1/admin/mcp-servers', {metadata: { name: connectorName },@@ -294,6 +303,7 @@ test.describe('optional QA recorder: Control UI connector edit', () => {},})expect(srvRes.status, `create connector: ${JSON.stringify(srvRes.data)}`).toBeLessThan(300)+ connectorCreated = trueawait page.getByRole('link', { name: 'Installed Connectors', exact: true }).click()await expect(page).toHaveURL(/\/connectors$/, { timeout: 20_000 })@@ -364,27 +374,70 @@ test.describe('optional QA recorder: Control UI connector edit', () => {await expect(page.getByText(/Rotation failed:/)).toHaveCount(0)await screenshotAndLog(page, testInfo, 'control-ui-connector-credential-rotation')+ } catch (err) {+ primaryFailure = err+ throw err} finally {- const deleteConnector = await directApi(- page.request,- 'DELETE',- `/api/v1/admin/mcp-servers/${encodeURIComponent(connectorName)}`- )- expect(deleteConnector.status, 'cleanup connector').toBe(200)- if (!cleanupIdentity) throw new Error('missing cleanup identity')- const deleteSecret = await directApi(- page.request,- 'DELETE',- `/api/v1/admin/mcp-secrets/${encodeURIComponent(secretName)}`,- cleanupIdentity- )- expect(deleteSecret.status, 'cleanup connector credential').toBe(200)- const deleteContext = await directApi(- page.request,- 'DELETE',- `/api/v1/admin/contexts/${encodeURIComponent(contextName)}`- )- expect(deleteContext.status, 'cleanup context').toBe(200)+ // A `throw` or a failed `expect` inside `finally` REPLACES the exception+ // the body raised, so a run that died at "create Secret" used to be+ // reported as "cleanup connector: expected 200, received 404". Collect+ // every cleanup problem instead, and let it fail the test only when the+ // body itself passed; otherwise log it next to the real failure.+ const cleanupFailures: string[] = []+ // Order matters: the connector references the Secret, and the MCP+ // Secret DELETE refuses a Secret a live connector still uses.+ if (connectorCreated) {+ const deleteConnector = await directApi(+ page.request,+ 'DELETE',+ `/api/v1/admin/mcp-servers/${encodeURIComponent(connectorName)}`+ )+ if (deleteConnector.status !== 200) {+ cleanupFailures.push(`connector ${connectorName}: HTTP ${deleteConnector.status}`)+ }+ }+ if (secretCreated) {+ if (!cleanupIdentity) {+ // The Secret exists but no CAS identity was captured: a bodyless+ // delete is exactly the unfenced path these journeys exist to+ // retire, so leave the object for a manual, identity-checked delete.+ cleanupFailures.push(+ `Secret ${secretName}: created without a CAS identity; delete it by hand`+ )+ } else {+ const deleteSecret = await directApi(+ page.request,+ 'DELETE',+ `/api/v1/admin/mcp-secrets/${encodeURIComponent(secretName)}`,+ cleanupIdentity+ )+ if (deleteSecret.status !== 200) {+ cleanupFailures.push(`Secret ${secretName}: HTTP ${deleteSecret.status}`)+ }+ }+ }+ if (contextCreated) {+ const deleteContext = await directApi(+ page.request,+ 'DELETE',+ `/api/v1/admin/contexts/${encodeURIComponent(contextName)}`+ )+ if (deleteContext.status !== 200) {+ cleanupFailures.push(`context ${contextName}: HTTP ${deleteContext.status}`)+ }+ }+ if (cleanupFailures.length > 0) {+ const message = `cleanup failed: ${cleanupFailures.join('; ')}`+ if (primaryFailure) {+ console.error(+ `${message} (after test failure: ${+ primaryFailure instanceof Error ? primaryFailure.message : String(primaryFailure)+ })`+ )+ } else {+ throw new Error(message)+ }+ }}})})diff --git a/control-ui/e2e/qa-recorder-connector-rotation-demo.spec.ts b/control-ui/e2e/qa-recorder-connector-rotation-demo.spec.tsindex fb4c86167..a916038aa 100644--- a/control-ui/e2e/qa-recorder-connector-rotation-demo.spec.ts+++ b/control-ui/e2e/qa-recorder-connector-rotation-demo.spec.ts@@ -9,7 +9,7 @@// DeploymentReady=True (not the tolerated timeout). Recorded via the// human-e2e-recorder skill (headed, slowMo, video). Opt-in: QA_RECORDER_CONFIRM_MUTATIONS=1.import { expect, test } from '@playwright/test'-import { requireSecretIdentity, type SecretIdentity } from '../test-utils/secretIdentity'+import { type SecretIdentity, requireSecretIdentity } from '../test-utils/secretIdentity'import {CONTROL_API_URL,CONTROL_UI_URL,@@ -36,6 +36,11 @@ test.describe('demo: Control UI credential-rotation journey', () => {const credentials = adminCredentials()let cleanupIdentity: SecretIdentity | null = null+ // Cleanup deletes only what this run actually created; see the finally.+ let secretCreated = false+ let contextCreated = false+ let connectorCreated = false+ let primaryFailure: unknown = nullconst contextName = uniqueE2EName('demo-context')const connectorName = uniqueE2EName('demo-connector')const secretName = uniqueE2EName('demo-secret')@@ -54,6 +59,7 @@ test.describe('demo: Control UI credential-rotation journey', () => {data: { [secretKey]: 'initial-value-123' },})expect(s.status, `Secret: ${JSON.stringify(s.data)}`).toBeLessThan(300)+ secretCreated = truecleanupIdentity = requireSecretIdentity(s.data, 'stage cleanup identity')const c = await api(page.request, 'POST', '/api/v1/admin/contexts', {@@ -61,6 +67,7 @@ test.describe('demo: Control UI credential-rotation journey', () => {spec: { contextId: contextName, description: 'demo rotation context', mcpServers: [] },})expect(c.status, `context: ${JSON.stringify(c.data)}`).toBeLessThan(300)+ contextCreated = trueconst srv = await api(page.request, 'POST', '/api/v1/admin/mcp-servers', {metadata: { name: connectorName },spec: {@@ -78,6 +85,7 @@ test.describe('demo: Control UI credential-rotation journey', () => {},})expect(srv.status, `connector: ${JSON.stringify(srv.data)}`).toBeLessThan(300)+ connectorCreated = true})await test.step('navigate: sidebar "Connectors" -> connectors list', async () => {@@ -151,27 +159,67 @@ test.describe('demo: Control UI credential-rotation journey', () => {// Linger on the success banner so the recording clearly shows it.await humanPause(page, 3000, 4000)})+ } catch (err) {+ primaryFailure = err+ throw err} finally {- const deleteConnector = await api(- page.request,- 'DELETE',- `/api/v1/admin/mcp-servers/${encodeURIComponent(connectorName)}`- )- expect(deleteConnector.status, 'cleanup connector').toBe(200)- if (!cleanupIdentity) throw new Error('missing cleanup identity')- const deleteSecret = await api(- page.request,- 'DELETE',- `/api/v1/admin/mcp-secrets/${encodeURIComponent(secretName)}`,- cleanupIdentity- )- expect(deleteSecret.status, 'cleanup connector credential').toBe(200)- const deleteContext = await api(- page.request,- 'DELETE',- `/api/v1/admin/contexts/${encodeURIComponent(contextName)}`- )- expect(deleteContext.status, 'cleanup context').toBe(200)+ // A `throw` or a failed `expect` inside `finally` REPLACES the exception+ // the body raised, so a run that died while staging used to be reported+ // as "cleanup connector: expected 200, received 404". Collect every+ // cleanup problem instead, and let it fail the test only when the body+ // itself passed; otherwise log it next to the real failure.+ const cleanupFailures: string[] = []+ // Order matters: the connector references the Secret, and the MCP+ // Secret DELETE refuses a Secret a live connector still uses.+ if (connectorCreated) {+ const deleteConnector = await api(+ page.request,+ 'DELETE',+ `/api/v1/admin/mcp-servers/${encodeURIComponent(connectorName)}`+ )+ if (deleteConnector.status !== 200) {+ cleanupFailures.push(`connector ${connectorName}: HTTP ${deleteConnector.status}`)+ }+ }+ if (secretCreated) {+ if (!cleanupIdentity) {+ cleanupFailures.push(+ `Secret ${secretName}: created without a CAS identity; delete it by hand`+ )+ } else {+ const deleteSecret = await api(+ page.request,+ 'DELETE',+ `/api/v1/admin/mcp-secrets/${encodeURIComponent(secretName)}`,+ cleanupIdentity+ )+ if (deleteSecret.status !== 200) {+ cleanupFailures.push(`Secret ${secretName}: HTTP ${deleteSecret.status}`)+ }+ }+ }+ if (contextCreated) {+ const deleteContext = await api(+ page.request,+ 'DELETE',+ `/api/v1/admin/contexts/${encodeURIComponent(contextName)}`+ )+ if (deleteContext.status !== 200) {+ cleanupFailures.push(`context ${contextName}: HTTP ${deleteContext.status}`)+ }+ }+ if (cleanupFailures.length > 0) {+ const message = `cleanup failed: ${cleanupFailures.join('; ')}`+ if (primaryFailure) {+ console.error(+ `${message} (after test failure: ${+ primaryFailure instanceof Error ? primaryFailure.message : String(primaryFailure)+ })`+ )+ } else {+ throw new Error(message)+ }+ }}})})diff --git a/control-ui/lib/__tests__/mcpIdentity.test.ts b/control-ui/lib/__tests__/mcpIdentity.test.tsindex 02a191a8e..62d9d3339 100644--- a/control-ui/lib/__tests__/mcpIdentity.test.ts+++ b/control-ui/lib/__tests__/mcpIdentity.test.ts@@ -97,8 +97,10 @@ describe('MCP identity compatibility', () => {))+ // The message must name the object left behind: the Secret exists on the+ // cluster and the UI will not fence a rollback on half an identity.await expect(createMcpSecret('partial-object', { fixture: 'fixture' })).rejects.toThrow(- /incomplete Secret identity/+ /incomplete Secret identity for Secret "partial-object" in mcp-server; repair is required/)})})diff --git a/control-ui/lib/api.ts b/control-ui/lib/api.tsindex 93a2a214b..eac3b079e 100644--- a/control-ui/lib/api.ts+++ b/control-ui/lib/api.ts@@ -14,7 +14,12 @@ export type SecretIdentity = {}function readSecretIdentity(raw: unknown, operation: string): SecretIdentity | undefined {- const value = (raw ?? {}) as { uid?: unknown; resourceVersion?: unknown }+ const value = (raw ?? {}) as {+ name?: unknown+ namespace?: unknown+ uid?: unknown+ resourceVersion?: unknown+ }const hasUid = value.uid !== undefinedconst hasResourceVersion = value.resourceVersion !== undefinedif (!hasUid && !hasResourceVersion) return undefined@@ -24,7 +29,18 @@ function readSecretIdentity(raw: unknown, operation: string): SecretIdentity | utypeof value.resourceVersion !== 'string' ||!value.resourceVersion.trim()) {- throw new Error(`${operation} returned an incomplete Secret identity; repair is required`)+ // This error is the operator's only pointer to the object that now needs+ // manual repair: the Secret WAS created, but without a complete CAS+ // identity the UI refuses to roll it back. Name it so "repair" is actionable.+ const subject =+ typeof value.name === 'string' && value.name+ ? ` for Secret "${value.name}"${+ typeof value.namespace === 'string' && value.namespace ? ` in ${value.namespace}` : ''+ }`+ : ''+ throw new Error(+ `${operation} returned an incomplete Secret identity${subject}; repair is required`+ )}return { uid: value.uid, resourceVersion: value.resourceVersion }}diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex da2202e58..5e68a1e21 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.499",+ "version": "0.1.500","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.499",+ "version": "0.1.500","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/llm-providers": "file:../packages/llm-providers",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 39f181a43..da72fd0cf 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.499",+ "version": "0.1.500","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/docs/how-to/control-api-mcp-secret-rollback-permits.md b/docs/how-to/control-api-mcp-secret-rollback-permits.mdindex 683ee08cc..793699ec3 100644--- a/docs/how-to/control-api-mcp-secret-rollback-permits.md+++ b/docs/how-to/control-api-mcp-secret-rollback-permits.md@@ -31,20 +31,40 @@ does not alter existing rows or tables.always carries UID and resourceVersion preconditions.- Expired rows are authorization-inert even before bounded physical cleanup.-## Image rollback after migration 0101+## Deploy order++Run the database-migration job **before** rolling the Control API image. The+Control API deployment is `strategy: Recreate` with a single replica+(`deploy/base/control-plane/control-api.yaml`), so the old pod is terminated+before the new one starts. A new image deployed against a database that has not+yet applied `0109` fails `assertDbReady` on startup with+`missing migrations 0109_mcp_secret_rollback_permits` — and because the previous+pod is already gone, that is a full control-plane outage, not a stalled+rollout. The CRD apply is independent of this ordering: control-api reads the+Secret-reference contract from a compiled-in constant, not from the CRD+annotation, so the charts may be applied before or after the image.++## Image rollback after migration 0109Rollback is **image-only**:-1. Keep migration `0101`, the new table, and the current runtime-access profile.+1. Keep migration `0109`, the new table, and the current runtime-access profile.2. Use the current checkout's deployment and database-migration tooling.3. Change only the Control API image to the previously verified image.4. Re-run the current runtime-access reconciliation and exact privilege check.The previous Control API binary ignores the additive table. Do not run an old-checkout's exact-schema verifier after `0101`; it does not know the new public+checkout's exact-schema verifier after `0109`; it does not know the new publicrelation and will correctly refuse to certify it. Never drop the table as partof an application-image rollback.+Rolling the **checkout** back is not a supported rollback path. The privilege+verifier is a full outer join between the repository's runtime-access profile+list and live `pg_class`; an older checkout's list omits+`mcp_secret_rollback_permits`, which the forward-only migration leaves in place,+so the verifier reports a relation-coverage violation and aborts the deploy. If+a runbook says "redeploy the previous tag", amend it before this ships.+## Failure outcomes- Permit persistence failure after Kubernetes create: `503 repair_required`