Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): count uninstall teardown failures as a metric (R2-L2)
Diego Zacariasffd075c6a62c
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex f799c7af2..b57504bc4 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.649",+ "version": "0.1.650","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.649",+ "version": "0.1.650","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex be4b5b6ca..a8d045a66 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.649",+ "version": "0.1.650","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 9d2abaae9..0563c285e 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -189,6 +189,17 @@ export const mcpHostJwtReissueTotal = getOrCreateCounter({labelNames: ['result'] as const as Array<'result'>,})+// MCP server uninstall teardown: a best-effort cleanup step failed AFTER the CR was+// already deleted (orphan oauth-client Secret, dynamic client row, or oauth_grants).+// Nothing retries once the CR is gone, so the structured error log is the only trace+// today — this counter makes the residual reconcilable/alertable. Rate > 0 ⇒ orphaned+// state to sweep. Stages: oauth_client_secret | dynamic_client | oauth_grants.+export const mcpServerUninstallTeardownFailuresTotal = getOrCreateCounter({+ name: 'mcp_server_uninstall_teardown_failures_total',+ help: 'Count of best-effort MCP server uninstall teardown steps that failed after CR deletion, by stage.',+ labelNames: ['stage'] as const as Array<'stage'>,+})+// ─── HTTP counters / histograms (scoped to workflow-approvals endpoints) ──export const mcpHostHttpTotal = getOrCreateCounter({name: 'mcp_host_http_total',diff --git a/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.tsnew file mode 100644index 000000000..ce2c631a8--- /dev/null+++ b/control-api/src/routes/admin/__tests__/mcpServerUninstallTeardownMetrics.test.ts@@ -0,0 +1,79 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'++/**+ * R2-L2 (T3/T4): a best-effort uninstall teardown step that fails AFTER the CR is+ * deleted increments mcp_server_uninstall_teardown_failures_total{stage} — the+ * signal that makes orphaned Secret / dynamic-client / grants state reconcilable+ * and alertable. The uninstall still returns 200: teardown never blocks the delete.+ *+ * Seam: the same pool.query swap as mcpServerUninstallPurgesGrants.test.ts, but the+ * oauth_grants DELETE throws so the H-3 catch runs. The counter is read from the+ * shared registry BY NAME (never imported as a symbol), so against a parent without+ * the metric the lookup returns 0 and the delta assertion fails — the T3 red —+ * rather than a compile error.+ */++const dbMock = vi.hoisted(() => {+ const query = vi.fn(async (text: string) => {+ if (/DELETE\s+FROM\s+oauth_grants/i.test(text)) {+ throw new Error('simulated oauth_grants purge failure')+ }+ return { rows: [], rowCount: 0 }+ })+ return { query }+})++vi.mock('../../../db.js', async importActual => {+ const actual = await importActual<typeof import('../../../db.js')>()+ return { ...actual, pool: { query: dbMock.query } }+})++// Import AFTER vi.mock so the swapped `pool` wires into the router.+const { createAdminResourcesRouter } = await import('../resources.js')+const { registry } = await import('../../../observability/metrics.js')++async function teardownFailures(stage: string): Promise<number> {+ const all = await registry.getMetricsAsJSON()+ const metric = all.find(m => m.name === 'mcp_server_uninstall_teardown_failures_total')+ if (!metric) return 0+ return (metric.values as Array<{ labels: Record<string, string>; value: number }>)+ .filter(v => v.labels.stage === stage)+ .reduce((sum, v) => sum + v.value, 0)+}++describe('DELETE /admin/mcp-servers/:name — teardown failure increments the counter (R2-L2)', () => {+ const SERVER_NAME = 'gdrive'++ function buildApp() {+ const gateway = {+ deleteResource: vi.fn(async () => ({ metadata: { name: SERVER_NAME } })),+ deleteSecret: vi.fn(async () => ({})),+ getSecret: vi.fn(async () => ({})),+ listResource: vi.fn(async () => []),+ updateResource: vi.fn(async () => ({})),+ }+ const app = express()+ app.use(express.json({ limit: '1mb' }))+ app.use('/api/v1', createAdminResourcesRouter(gateway as never))+ return { app }+ }++ beforeEach(() => {+ dbMock.query.mockClear()+ })++ it('counts a failed oauth_grants purge under stage=oauth_grants and still returns 200', async () => {+ const before = await teardownFailures('oauth_grants')+ const { app } = buildApp()++ const res = await request(app).delete(`/api/v1/admin/mcp-servers/${SERVER_NAME}`)++ // Teardown is best-effort — a purge failure must never block the uninstall.+ expect(res.status).toBe(200)+ // The failed stage is now an observable metric, not just a log line.+ const after = await teardownFailures('oauth_grants')+ expect(after - before).toBeGreaterThanOrEqual(1)+ })+})diff --git a/control-api/src/routes/admin/resources.ts b/control-api/src/routes/admin/resources.tsindex 2b590f5f7..2ee6b622c 100644--- a/control-api/src/routes/admin/resources.ts+++ b/control-api/src/routes/admin/resources.ts@@ -16,6 +16,7 @@ import { cleanupDynamicClientForServer } from '../../oauth/dcrCleanup.js'import { deriveOAuthEncryptionKey } from '../../oauth/encryption.js'import { deleteOAuthGrantsForServer } from '../../oauth/store.js'import { rootLogger } from '../../observability/logger.js'+import { mcpServerUninstallTeardownFailuresTotal } from '../../observability/metrics.js'import { stripHookRefFromHosts } from '../../services/hostGuardrailRefs.js'import {K8sConflictError,@@ -1251,6 +1252,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {'MCP OAuth client Secret already gone')} else {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_client_secret' })log.error({ secretName: mcpOAuthClientSecretName, namespace: ns, err },'McpServer delete succeeded but OAuth client Secret cleanup failed'@@ -1301,6 +1303,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {)}} catch (err) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'dynamic_client' })log.error({ serverName: name, namespace: ns, err },'Dynamic client cleanup failed on uninstall (CR already deleted)'@@ -1323,6 +1326,7 @@ export function createAdminResourcesRouter(gateway: K8sGateway): Router {)}} catch (err) {+ mcpServerUninstallTeardownFailuresTotal.inc({ stage: 'oauth_grants' })log.error({ serverName: name, namespace: ns, err },'OAuth grants purge failed on uninstall (CR already deleted)'