Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): fence remote-install saga rollback deletes with uid/resourceVersion

Diego Zacarias70724c59e852
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index c590eda6b..f95b80ddd 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.644",
+ "version": "0.1.645",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.644",
+ "version": "0.1.645",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index f6e9dac46..f86280923 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.644",
+ "version": "0.1.645",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/admin/remoteMcp.ts b/control-api/src/routes/admin/remoteMcp.ts
index a59ad3724..b747f9a54 100644
--- a/control-api/src/routes/admin/remoteMcp.ts
+++ b/control-api/src/routes/admin/remoteMcp.ts
@@ -27,6 +27,8 @@ import {
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { rootLogger } from '../../observability/logger.js'
import { K8sNotFoundError } from '../../services/resourceService.js'
+import type { SecretSnapshot } from '../../services/secretRepository.js'
+import type { ResourcePreconditions } from '../../types.js'
import { normalizeConfiguredOrigin } from '../external/oauthCallback.js'
/**
@@ -87,6 +89,24 @@ function isValidK8sName(name: string): boolean {
return RFC1123_RE.test(name) && name.length <= 253
}
+/**
+ * Fenced-delete preconditions from a just-created object, or undefined when the
+ * apiserver response carried no complete identity. A rollback bound by
+ * uid+resourceVersion cannot arrase a homonymous object that a concurrent
+ * uninstall+reinstall recreated in the compensation window — the fenced pattern
+ * the baked carril (`registry.ts`) and uninstall (`resources.ts`) already use.
+ */
+function resourcePreconditionsFrom(resource: unknown): ResourcePreconditions | undefined {
+ const metadata = (resource as { metadata?: { uid?: unknown; resourceVersion?: unknown } } | null)
+ ?.metadata
+ const uid = metadata?.uid
+ const resourceVersion = metadata?.resourceVersion
+ if (typeof uid !== 'string' || !uid || typeof resourceVersion !== 'string' || !resourceVersion) {
+ return undefined
+ }
+ return { uid, resourceVersion }
+}
+
/**
* The `spec.oauth` remote shape written to the CR — the C3 contract. Uses a distinct
* `source: 'remote'` discriminator instead of the closed 8-provider `provider` enum
@@ -664,10 +684,15 @@ export function createAdminRemoteMcpRouter(
}
// ── Saga step 1: create the client Secret (confidential only) ─────────
- let secretCreated = false
+ // Fenced rollback (P1): capture the created objects' server identities so
+ // every compensating delete is bound by uid+resourceVersion. A by-name
+ // delete would arrase a Secret/CR that a concurrent uninstall+reinstall of
+ // the same serverName recreated between the create and the rollback.
+ let createdClientSecretSnapshot: SecretSnapshot | null = null
+ let createdServerPreconditions: ResourcePreconditions | undefined
if (effectiveClientMode === 'confidential' && clientSecretName) {
try {
- await gateway.createSecret({
+ createdClientSecretSnapshot = await gateway.createSecret({
name: clientSecretName,
namespace: targetNs,
type: 'Opaque',
@@ -685,7 +710,6 @@ export function createAdminRemoteMcpRouter(
}
throw err
}
- secretCreated = true
// Names-only audit: never echo the secret material.
log.info(
{
@@ -699,15 +723,19 @@ export function createAdminRemoteMcpRouter(
// ── Saga step 2: create the McpServer CR (rollback Secret on failure) ──
try {
- await gateway.createResource(
+ const createdServer = await gateway.createResource(
'mcpservers',
{ metadata: { name: serverName, labels: managedLabels }, spec: mcpServerSpec },
targetNs
)
+ createdServerPreconditions = resourcePreconditionsFrom(createdServer)
} catch (err) {
- if (secretCreated && clientSecretName) {
+ if (createdClientSecretSnapshot && clientSecretName) {
try {
- await gateway.deleteSecret(clientSecretName, targetNs)
+ await gateway.deleteSecret(clientSecretName, targetNs, {
+ uid: createdClientSecretSnapshot.uid,
+ resourceVersion: createdClientSecretSnapshot.resourceVersion,
+ })
} catch {
// Best-effort rollback; preserve the original CR-create error.
}
@@ -739,7 +767,12 @@ export function createAdminRemoteMcpRouter(
}
} catch (err) {
try {
- await gateway.deleteResource('mcpservers', serverName, targetNs)
+ await gateway.deleteResource(
+ 'mcpservers',
+ serverName,
+ targetNs,
+ createdServerPreconditions
+ )
await waitForDeletion(
() => gateway.getResource('mcpservers', serverName, targetNs),
`McpServer/${serverName}`
@@ -747,9 +780,12 @@ export function createAdminRemoteMcpRouter(
} catch {
// Best-effort rollback; preserve the original attach error.
}
- if (secretCreated && clientSecretName) {
+ if (createdClientSecretSnapshot && clientSecretName) {
try {
- await gateway.deleteSecret(clientSecretName, targetNs)
+ await gateway.deleteSecret(clientSecretName, targetNs, {
+ uid: createdClientSecretSnapshot.uid,
+ resourceVersion: createdClientSecretSnapshot.resourceVersion,
+ })
await waitForDeletion(
() => gateway.getSecret(clientSecretName, targetNs),
`Secret/${clientSecretName}`
diff --git a/control-api/test/routes.adminRemoteMcp.test.ts b/control-api/test/routes.adminRemoteMcp.test.ts
index 8680b7f6e..91da04d0c 100644
--- a/control-api/test/routes.adminRemoteMcp.test.ts
+++ b/control-api/test/routes.adminRemoteMcp.test.ts
@@ -331,6 +331,85 @@ describe('POST /admin/mcp-servers/remote (install saga)', () => {
await expect(gw.getSecret('attach-remote-oauth-client', NS)).rejects.toThrow()
})
+ it('fenced Secret rollback: a homonym recreated with a NEW uid survives the CR-create rollback (P1)', async () => {
+ mockDiscovery(notionResult)
+ const gw = gatewayWithContext('ctx-a')
+ const secretName = 'race-remote-oauth-client'
+
+ // Force step-2 (CR create) to fail, but first simulate a concurrent
+ // uninstall+reinstall of the same serverName: delete the just-created client
+ // Secret and recreate a homonym carrying a DIFFERENT uid. `createSecret` is
+ // the real producer (T1) — the mock's allocateUid is monotonic, so the
+ // recreated Secret's uid necessarily differs from the one the saga captured.
+ let recreatedUid: string | undefined
+ vi.spyOn(gw, 'createResource').mockImplementationOnce(async () => {
+ await gw.deleteSecret(secretName, NS)
+ const recreated = await gw.createSecret({
+ name: secretName,
+ namespace: NS,
+ type: 'Opaque',
+ stringData: { client_id: 'other-tenant', client_secret: 'other-secret' },
+ })
+ recreatedUid = recreated.uid
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ })
+
+ const res = await request(makeApp(gw)).post('/admin/mcp-servers/remote').send({
+ serverName: 'race-remote',
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'pre-registered',
+ clientId: 'cid',
+ clientSecret: 'csecret',
+ })
+
+ expect(res.status).toBe(409)
+ // The homonym belongs to the concurrent install: the fenced rollback's uid
+ // precondition rejects the delete (409, swallowed best-effort), so the other
+ // tenant's credentials survive intact. A by-name rollback would arrase them.
+ const survivor = await gw.getSecret(secretName, NS)
+ expect(survivor.metadata.uid).toBe(recreatedUid)
+ })
+
+ it('fenced CR rollback: a homonym recreated with a NEW uid survives the Context-attach rollback (P1)', async () => {
+ mockDiscovery(notionResult)
+ const gw = gatewayWithContext('ctx-a')
+ const serverName = 'race2-remote'
+
+ // Let the CR create succeed, then fail the Context attach (updateResource).
+ // Between the two, simulate a concurrent uninstall+reinstall that replaces
+ // the McpServer CR with a homonym carrying a NEW uid (createResource is the
+ // real producer, T1). A by-name rollback would delete the wrong object.
+ let recreatedCrUid: string | undefined
+ vi.spyOn(gw, 'updateResource').mockImplementationOnce(async () => {
+ await gw.deleteResource('mcpservers', serverName, NS)
+ const recreated = (await gw.createResource(
+ 'mcpservers',
+ { metadata: { name: serverName }, spec: { image: 'other' } },
+ NS
+ )) as { metadata: { uid: string } }
+ recreatedCrUid = recreated.metadata.uid
+ throw Object.assign(new Error('conflict'), { code: 409 })
+ })
+
+ const res = await request(makeApp(gw)).post('/admin/mcp-servers/remote').send({
+ serverName,
+ contextRef: 'ctx-a',
+ baseUrl: 'https://mcp.notion.com/mcp',
+ mode: 'pre-registered',
+ clientId: 'cid',
+ clientSecret: 'csecret',
+ })
+
+ expect(res.status).toBeGreaterThanOrEqual(400)
+ // The recreated CR (a different uid) must survive: the fenced deleteResource
+ // precondition rejects the compensating delete instead of arrasing it.
+ const survivor = (await gw.getResource('mcpservers', serverName, NS)) as {
+ metadata: { uid: string }
+ }
+ expect(survivor.metadata.uid).toBe(recreatedCrUid)
+ })
+
it('forces the namespace server-side (caller-supplied namespace is ignored)', async () => {
mockDiscovery(notionResult)
const gw = gatewayWithContext('ctx-a')