Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): label host-binding outcomes in workflow_auth_refresh_total (BR-04)
Diego Zacarias2d761f0e38ce
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 27699aba0..a58ec0225 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.823",+ "version": "0.1.824","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.823",+ "version": "0.1.824","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 96964cb2f..7b6dd017b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.823",+ "version": "0.1.824","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 049239098..060edb03b 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -188,10 +188,21 @@ export const oauthBrokerJwtIssueTotal = getOrCreateCounter({labelNames: ['result'] as const as Array<'result'>, // issued | recipe_not_found | not_background_access})+// Refresh endpoint outcomes (also the label of the duration histogram below).+// The 401 body stays opaque; the label is emitted server-side in a `finally`.+// Label values, aligned with the re-issue counter where the outcome is shared:+// success — token rotated, fresh token set minted+// failed — missing bearer; signature/claims/expiry rejected or+// jti already consumed at verify time; unhandled error+// host_unavailable — Host lookup failed transiently (503, jti not consumed)+// host_mismatch — HCC chain's Host is gone or was recreated (new uid)+// lineage_without_audience — hosts-namespace standalone chain without HCC audience+// revoked — jti consumed concurrently between verify and consume+// Rate-limited requests never reach the handler and are not counted here.export const mcpHostJwtRefreshTotal = getOrCreateCounter({name: 'workflow_auth_refresh_total',help: 'Count of workflow refresh attempts labelled by result.',- labelNames: ['result'] as const as Array<'result'>, // success | failed | rotated+ labelNames: ['result'] as const as Array<'result'>,})export const mcpHostJwtRefreshDurationSeconds = getOrCreateHistogram({@@ -212,8 +223,11 @@ export const mcpHostJwtRefreshDurationSeconds = getOrCreateHistogram({// expired_beyond_reissue_grace — signed refresh token is too old to reissue// revoked — jti already present in workflow_revoked_refresh_jtis// mismatch — recipe_name body did not match JWT sub's recipe component-// rate_limited — limiter denied before handler ran (emitted in middleware)// bad_request — missing/malformed body or bearer token+// host_unavailable — Host lookup failed transiently (503, jti not consumed)+// host_mismatch — HCC chain's Host is gone or was recreated (new uid)+// lineage_without_audience — hosts-namespace standalone chain without HCC audience+// Rate-limited requests never reach the handler and are not counted here.export const mcpHostJwtReissueTotal = getOrCreateCounter({name: 'workflow_auth_reissue_requests_total',help: 'Count of workflow auth re-issue attempts labelled by result.',diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.tsindex 8330df358..fe28ab65f 100644--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts@@ -37,7 +37,15 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Router(req, res, next) => {void (async () => {const refreshStart = process.hrtime.bigint()- let resultLabel: 'success' | 'failed' = 'failed'+ // Values mirror the reissue route's labels so the two can be compared.+ type ResultLabel =+ | 'success'+ | 'failed'+ | 'host_unavailable'+ | 'host_mismatch'+ | 'lineage_without_audience'+ | 'revoked'+ let resultLabel: ResultLabel = 'failed'try {const token = extractBearerToken(req)if (!token) {@@ -65,6 +73,7 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Routerconst binding = await resolveRenewalHostBinding(gateway, claims)if (binding.kind === 'unavailable') {+ resultLabel = 'host_unavailable'req.log?.warn({ event: 'host_state_unavailable', route: 'workflow_auth_refresh' },'host lookup failed; refresh token left unconsumed'@@ -72,6 +81,10 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Routerreturn res.status(503).json({ error: 'host_state_unavailable' })}if (binding.kind === 'denied') {+ resultLabel =+ binding.reason === 'hcc_lineage_without_audience'+ ? 'lineage_without_audience'+ : 'host_mismatch'req.log?.info({ event: 'auth_denied', reason: binding.reason, route: 'workflow_auth_refresh' },'auth denied'@@ -81,6 +94,7 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Routerconst consumed = await consumeRefreshJti(claims.jti, claims.exp)if (!consumed) {+ resultLabel = 'revoked'req.log?.info({ event: 'auth_denied', reason: 'jti_race_lost', route: 'workflow_auth_refresh' },'auth denied'diff --git a/control-api/test/routes.mcp-host.user-approval-requests.test.ts b/control-api/test/routes.mcp-host.user-approval-requests.test.tsindex c8d863a16..4292d6da7 100644--- a/control-api/test/routes.mcp-host.user-approval-requests.test.ts+++ b/control-api/test/routes.mcp-host.user-approval-requests.test.ts@@ -3,6 +3,7 @@ import jwt from 'jsonwebtoken'import request from 'supertest'import { createApp } from '../src/app.js'import { config } from '../src/config.js'+import { mcpHostJwtRefreshTotal } from '../src/observability/metrics.js'import * as notificationDeliveryQueueService from '../src/services/notificationDeliveryQueueService.js'import * as userApprovalRequestService from '../src/services/userApprovalRequestService.js'import * as workflowApprovalMediumLinkSessionService from '../src/services/workflowApprovalMediumLinkSessionService.js'@@ -59,6 +60,8 @@ vi.mock('../src/services/notificationEmitter.js', () => ({// CONFLICT DO NOTHING = consume) so the real verify-then-consume path runs.const consumedRefreshJtis = new Set<string>()let failNextRefreshJtiInsert: Error | null = null+// Another request consumed the jti between this request's verify and consume.+let conflictOnNextRefreshJtiInsert = falsevi.mock('../src/db.js', () => ({pool: {query: vi.fn(async (sql: unknown, params?: unknown[]) => {@@ -74,6 +77,11 @@ vi.mock('../src/db.js', () => ({failNextRefreshJtiInsert = nullthrow err}+ if (conflictOnNextRefreshJtiInsert) {+ conflictOnNextRefreshJtiInsert = false+ consumedRefreshJtis.add(jti)+ return { rows: [], rowCount: 0 }+ }if (consumedRefreshJtis.has(jti)) return { rows: [], rowCount: 0 }consumedRefreshJtis.add(jti)return { rows: [{ jti }], rowCount: 1 }@@ -135,6 +143,7 @@ describe('User Approval Request Routes', () => {vi.clearAllMocks()consumedRefreshJtis.clear()failNextRefreshJtiInsert = null+ conflictOnNextRefreshJtiInsert = falsevi.mocked(userApprovalRequestService.parseWorkflowTriggerIntent).mockImplementation((payload: unknown) => {const record =@@ -295,6 +304,133 @@ describe('User Approval Request Routes', () => {expect(res.body.error).toBe('Internal Server Error')})+ describe('workflow_auth_refresh_total result label', () => {+ async function refreshCount(result: string): Promise<number> {+ const metric = await mcpHostJwtRefreshTotal.get()+ return metric.values.find(v => v.labels.result === result)?.value ?? 0+ }++ async function snapshot(): Promise<Record<string, number>> {+ const labels = [+ 'success',+ 'failed',+ 'host_unavailable',+ 'host_mismatch',+ 'lineage_without_audience',+ 'revoked',+ ]+ const out: Record<string, number> = {}+ for (const label of labels) out[label] = await refreshCount(label)+ return out+ }++ async function expectOnlyIncremented(before: Record<string, number>, label: string) {+ const after = await snapshot()+ for (const [key, value] of Object.entries(before)) {+ expect([key, after[key]]).toEqual([key, key === label ? value + 1 : value])+ }+ }++ async function seedSharedHost(): Promise<string> {+ const host = (await gateway.createResource(+ 'hosts',+ {+ metadata: { name: SHARED_HOST },+ spec: { host: SHARED_HOST, contextRef: 'ctx-a', secretRef: `${SHARED_HOST}-llm` },+ },+ SHARED_HOST_NS+ )) as { metadata: { uid: string } }+ return host.metadata.uid+ }++ function issueHccRefresh(hostUid: string): string {+ return mcpHostJwt.issueMcpHostRefreshJwt(SHARED_HOST_NS, 'standalone', [SHARED_HOST], {+ workflowControlScopes: ['workflow:list'],+ hccCredential: { hostUid },+ }).token+ }++ function postRefresh(token: string) {+ return request(app)+ .post('/api/v1/workflow-auth/refresh')+ .set('Authorization', `Bearer ${token}`)+ .send()+ }++ it('counts a transient Host lookup failure as host_unavailable', async () => {+ const token = issueHccRefresh(await seedSharedHost())+ gateway.failGetResource('hosts', { statusCode: 503 })+ const before = await snapshot()++ const res = await postRefresh(token)++ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'host_state_unavailable' })+ await expectOnlyIncremented(before, 'host_unavailable')+ })++ it('counts a chain whose Host was recreated with a new uid as host_mismatch', async () => {+ const oldUid = await seedSharedHost()+ const token = issueHccRefresh(oldUid)+ await gateway.deleteResource('hosts', SHARED_HOST, SHARED_HOST_NS)+ expect(await seedSharedHost()).not.toBe(oldUid)+ const before = await snapshot()++ const res = await postRefresh(token)++ expect(res.status).toBe(401)+ await expectOnlyIncremented(before, 'host_mismatch')+ })++ it('counts a chain whose Host no longer exists as host_mismatch', async () => {+ const hostUid = await seedSharedHost()+ const token = issueHccRefresh(hostUid)+ await gateway.deleteResource('hosts', SHARED_HOST, SHARED_HOST_NS)+ const before = await snapshot()++ const res = await postRefresh(token)++ expect(res.status).toBe(401)+ await expectOnlyIncremented(before, 'host_mismatch')+ })++ it('counts a hosts-namespace standalone chain without HCC audience as lineage_without_audience', async () => {+ await seedSharedHost()+ const token = mcpHostJwt.issueMcpHostRefreshJwt(+ SHARED_HOST_NS,+ 'standalone',+ [SHARED_HOST],+ { workflowControlScopes: ['workflow:list'] }+ ).token+ const before = await snapshot()++ const res = await postRefresh(token)++ expect(res.status).toBe(401)+ await expectOnlyIncremented(before, 'lineage_without_audience')+ })++ it('counts a jti consumed between verify and consume as revoked', async () => {+ const { refreshToken } = issueTestTokens()+ conflictOnNextRefreshJtiInsert = true+ const before = await snapshot()++ const res = await postRefresh(refreshToken)++ expect(res.status).toBe(401)+ await expectOnlyIncremented(before, 'revoked')+ })++ it('keeps an invalid refresh token on the failed label', async () => {+ const before = await snapshot()++ const res = await postRefresh('invalid-token')++ expect(res.status).toBe(401)+ await expectOnlyIncremented(before, 'failed')+ })+ })+it('rejects access token used as refresh', async () => {const { accessToken } = issueTestTokens()const res = await request(app)