Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): count uninstall teardown failures as a metric (R2-L2)

Diego Zacariasffd075c6a62c
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index f799c7af2..b57504bc4 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.649",
+ "version": "0.1.650",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.649",
+ "version": "0.1.650",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index be4b5b6ca..a8d045a66 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.649",
+ "version": "0.1.650",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 9d2abaae9..0563c285e 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -189,6 +189,17 @@ export const mcpHostJwtReissueTotal = getOrCreateCounter({
labelNames: ['result'] as const as Array<'result'>,
})
+// MCP server uninstall teardown: a best-effort cleanup step failed AFTER the CR was
+// already deleted (orphan oauth-client Secret, dynamic client row, or oauth_grants).
+// Nothing retries once the CR is gone, so the structured error log is the only trace
+// today — this counter makes the residual reconcilable/alertable. Rate > 0 ⇒ orphaned
+// state to sweep. Stages: oauth_client_secret | dynamic_client | oauth_grants.
+export const mcpServerUninstallTeardownFailuresTotal = getOrCreateCounter({
+ name: 'mcp_server_uninstall_teardown_failures_total',
+ help: 'Count of best-effort MCP server uninstall teardown steps that failed after CR deletion, by stage.',
+ labelNames: ['stage'] as const as Array<'stage'>,
+})
+
// ─── HTTP counters / histograms (scoped to workflow-approvals endpoints) ──
export const mcpHostHttpTotal = getOrCreateCounter({
name: 'mcp_host_http_total',
diff --git a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts
new file mode 100644
index 000000000..ce2c631a8
--- /dev/null
+++ b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts
@@ -0,0 +1,79 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+
+/**
+ * R2-L2 (T3/T4): a best-effort uninstall teardown step that fails AFTER the CR is
+ * deleted increments mcp_server_uninstall_teardown_failures_total{stage} — the
+ * signal that makes orphaned Secret / dynamic-client / grants state reconcilable
+ * and alertable. The uninstall still returns 200: teardown never blocks the delete.
+ *
+ * Seam: the same pool.query swap as mcpServerUninstallPurgesGrants.test.ts, but the
+ * oauth_grants DELETE throws so the H-3 catch runs. The counter is read from the
+ * shared registry BY NAME (never imported as a symbol), so against a parent without
+ * the metric the lookup returns 0 and the delta assertion fails — the T3 red —
+ * rather than a compile error.
+ */
+
+const dbMock = vi.hoisted(() => {
+ const query = vi.fn(async (text: string) => {
+ if (/DELETE\s+FROM\s+oauth_grants/i.test(text)) {
+ throw new Error('simulated oauth_grants purge failure')
+ }
+ return { rows: [], rowCount: 0 }
+ })
+ return { query }
+})
+
+vi.mock('../../../db.js', async importActual => {
+ const actual = await importActual<typeof import('../../../db.js')>()
+ return { ...actual, pool: { query: dbMock.query } }
+})
+
+// Import AFTER vi.mock so the swapped `pool` wires into the router.
+const { createAdminResourcesRouter } = await import('../resources.js')
+const { registry } = await import('../../../observability/metrics.js')
+
+async function teardownFailures(stage: string): Promise<number> {
+ const all = await registry.getMetricsAsJSON()
+ const metric = all.find(m => m.name === 'mcp_server_uninstall_teardown_failures_total')
+ if (!metric) return 0
+ return (metric.values as Array<{ labels: Record<string, string>; value: number }>)
+ .filter(v => v.labels.stage === stage)
+ .reduce((sum, v) => sum + v.value, 0)
+}
+
+describe('DELETE /admin/mcp-servers/:name — teardown failure increments the counter (R2-L2)', () => {
+ const SERVER_NAME = 'gdrive'
+
+ function buildApp() {
+ const gateway = {
+ deleteResource: vi.fn(async () => ({ metadata: { name: SERVER_NAME } })),
+ deleteSecret: vi.fn(async () => ({})),
+ getSecret: vi.fn(async () => ({})),
+ listResource: vi.fn(async () => []),
+ updateResource: vi.fn(async () => ({})),
+ }
+ const app = express()
+ app.use(express.json({ limit: '1mb' }))
+ app.use('/api/v1', createAdminResourcesRouter(gateway as never))
+ return { app }
+ }
+
+ beforeEach(() => {
+ dbMock.query.mockClear()
+ })
+
+ it('counts a failed oauth_grants purge under stage=oauth_grants and still returns 200', async () => {
+ const before = await teardownFailures('oauth_grants')
+ const { app } = buildApp()
+
+ const res = await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)
+
+ // Teardown is best-effort — a purge failure must never block the uninstall.
+ expect(res.status).toBe(200)
+ // The failed stage is now an observable metric, not just a log line.
+ const after = await teardownFailures('oauth_grants')
+ expect(after - before).toBeGreaterThanOrEqual(1)
+ })
+})
diff --git a/control-api/src/routes/admin/resources.ts b/control-api/src/routes/admin/resources.ts
index 2b590f5f7..2ee6b622c 100644
--- a/control-api/src/routes/admin/resources.ts
+++ b/control-api/src/routes/admin/resources.ts
@@ -16,6 +16,7 @@ import { cleanupDynamicClientForServer } from '../../oauth/dcrCleanup.js'
import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'
import { deleteOAuthGrantsForServer } from '../../oauth/store.js'
import { rootLogger } from '../../observability/logger.js'
+import { mcpServerUninstallTeardownFailuresTotal } from '../../observability/metrics.js'
import { stripHookRefFromHosts } from '../../services/hostGuardrailRefs.js'
import {
K8sConflictError,
@@ -1251,6 +1252,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
'MCP OAuth client Secret already gone'
)
} else {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_client_secret' })
log.error(
{ secretName: mcpOAuthClientSecretName, namespace: ns, err },
'McpServer delete succeeded but OAuth client Secret cleanup failed'
@@ -1301,6 +1303,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
)
}
} catch (err) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'dynamic_client' })
log.error(
{ serverName: name, namespace: ns, err },
'Dynamic client cleanup failed on uninstall (CR already deleted)'
@@ -1323,6 +1326,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {
)
}
} catch (err) {
+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants' })
log.error(
{ serverName: name, namespace: ns, err },
'OAuth grants purge failed on uninstall (CR already deleted)'