Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): label host-binding outcomes in workflow_auth_refresh_total (BR-04)

Diego Zacarias2d761f0e38ce
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 27699aba0..a58ec0225 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.823",
+ "version": "0.1.824",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.823",
+ "version": "0.1.824",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 96964cb2f..7b6dd017b 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.823",
+ "version": "0.1.824",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 049239098..060edb03b 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -188,10 +188,21 @@ export const oauthBrokerJwtIssueTotal = getOrCreateCounter({
labelNames: ['result'] as const as Array<'result'>, // issued | recipe_not_found | not_background_access
})
+// Refresh endpoint outcomes (also the label of the duration histogram below).
+// The 401 body stays opaque; the label is emitted server-side in a `finally`.
+// Label values, aligned with the re-issue counter where the outcome is shared:
+// success — token rotated, fresh token set minted
+// failed — missing bearer; signature/claims/expiry rejected or
+// jti already consumed at verify time; unhandled error
+// host_unavailable — Host lookup failed transiently (503, jti not consumed)
+// host_mismatch — HCC chain's Host is gone or was recreated (new uid)
+// lineage_without_audience — hosts-namespace standalone chain without HCC audience
+// revoked — jti consumed concurrently between verify and consume
+// Rate-limited requests never reach the handler and are not counted here.
export const mcpHostJwtRefreshTotal = getOrCreateCounter({
name: 'workflow_auth_refresh_total',
help: 'Count of workflow refresh attempts labelled by result.',
- labelNames: ['result'] as const as Array<'result'>, // success | failed | rotated
+ labelNames: ['result'] as const as Array<'result'>,
})
export const mcpHostJwtRefreshDurationSeconds = getOrCreateHistogram({
@@ -212,8 +223,11 @@ export const mcpHostJwtRefreshDurationSeconds = getOrCreateHistogram({
// expired_beyond_reissue_grace — signed refresh token is too old to reissue
// revoked — jti already present in workflow_revoked_refresh_jtis
// mismatch — recipe_name body did not match JWT sub's recipe component
-// rate_limited — limiter denied before handler ran (emitted in middleware)
// bad_request — missing/malformed body or bearer token
+// host_unavailable — Host lookup failed transiently (503, jti not consumed)
+// host_mismatch — HCC chain's Host is gone or was recreated (new uid)
+// lineage_without_audience — hosts-namespace standalone chain without HCC audience
+// Rate-limited requests never reach the handler and are not counted here.
export const mcpHostJwtReissueTotal = getOrCreateCounter({
name: 'workflow_auth_reissue_requests_total',
help: 'Count of workflow auth re-issue attempts labelled by result.',
diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
index 8330df358..fe28ab65f 100644
--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
@@ -37,7 +37,15 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Router
(req, res, next) => {
void (async () => {
const refreshStart = process.hrtime.bigint()
- let resultLabel: 'success' | 'failed' = 'failed'
+ // Values mirror the reissue route's labels so the two can be compared.
+ type ResultLabel =
+ | 'success'
+ | 'failed'
+ | 'host_unavailable'
+ | 'host_mismatch'
+ | 'lineage_without_audience'
+ | 'revoked'
+ let resultLabel: ResultLabel = 'failed'
try {
const token = extractBearerToken(req)
if (!token) {
@@ -65,6 +73,7 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Router
const binding = await resolveRenewalHostBinding(gateway, claims)
if (binding.kind === 'unavailable') {
+ resultLabel = 'host_unavailable'
req.log?.warn(
{ event: 'host_state_unavailable', route: 'workflow_auth_refresh' },
'host lookup failed; refresh token left unconsumed'
@@ -72,6 +81,10 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Router
return res.status(503).json({ error: 'host_state_unavailable' })
}
if (binding.kind === 'denied') {
+ resultLabel =
+ binding.reason === 'hcc_lineage_without_audience'
+ ? 'lineage_without_audience'
+ : 'host_mismatch'
req.log?.info(
{ event: 'auth_denied', reason: binding.reason, route: 'workflow_auth_refresh' },
'auth denied'
@@ -81,6 +94,7 @@ export function createMcpHostRefreshRoutes(gateway: RenewalHostGateway): Router
const consumed = await consumeRefreshJti(claims.jti, claims.exp)
if (!consumed) {
+ resultLabel = 'revoked'
req.log?.info(
{ event: 'auth_denied', reason: 'jti_race_lost', route: 'workflow_auth_refresh' },
'auth denied'
diff --git a/control-api/test/routes.mcp-host.user-approval-requests.test.ts b/control-api/test/routes.mcp-host.user-approval-requests.test.ts
index c8d863a16..4292d6da7 100644
--- a/control-api/test/routes.mcp-host.user-approval-requests.test.ts
+++ b/control-api/test/routes.mcp-host.user-approval-requests.test.ts
@@ -3,6 +3,7 @@ import jwt from 'jsonwebtoken'
import request from 'supertest'
import { createApp } from '../src/app.js'
import { config } from '../src/config.js'
+import { mcpHostJwtRefreshTotal } from '../src/observability/metrics.js'
import * as notificationDeliveryQueueService from '../src/services/notificationDeliveryQueueService.js'
import * as userApprovalRequestService from '../src/services/userApprovalRequestService.js'
import * as workflowApprovalMediumLinkSessionService from '../src/services/workflowApprovalMediumLinkSessionService.js'
@@ -59,6 +60,8 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
// CONFLICT DO NOTHING = consume) so the real verify-then-consume path runs.
const consumedRefreshJtis = new Set<string>()
let failNextRefreshJtiInsert: Error | null = null
+// Another request consumed the jti between this request's verify and consume.
+let conflictOnNextRefreshJtiInsert = false
vi.mock('../src/db.js', () => ({
pool: {
query: vi.fn(async (sql: unknown, params?: unknown[]) => {
@@ -74,6 +77,11 @@ vi.mock('../src/db.js', () => ({
failNextRefreshJtiInsert = null
throw err
}
+ if (conflictOnNextRefreshJtiInsert) {
+ conflictOnNextRefreshJtiInsert = false
+ consumedRefreshJtis.add(jti)
+ return { rows: [], rowCount: 0 }
+ }
if (consumedRefreshJtis.has(jti)) return { rows: [], rowCount: 0 }
consumedRefreshJtis.add(jti)
return { rows: [{ jti }], rowCount: 1 }
@@ -135,6 +143,7 @@ describe('User Approval Request Routes', () => {
vi.clearAllMocks()
consumedRefreshJtis.clear()
failNextRefreshJtiInsert = null
+ conflictOnNextRefreshJtiInsert = false
vi.mocked(userApprovalRequestService.parseWorkflowTriggerIntent).mockImplementation(
(payload: unknown) => {
const record =
@@ -295,6 +304,133 @@ describe('User Approval Request Routes', () => {
expect(res.body.error).toBe('Internal Server Error')
})
+ describe('workflow_auth_refresh_total result label', () => {
+ async function refreshCount(result: string): Promise<number> {
+ const metric = await mcpHostJwtRefreshTotal.get()
+ return metric.values.find(v => v.labels.result === result)?.value ?? 0
+ }
+
+ async function snapshot(): Promise<Record<string, number>> {
+ const labels = [
+ 'success',
+ 'failed',
+ 'host_unavailable',
+ 'host_mismatch',
+ 'lineage_without_audience',
+ 'revoked',
+ ]
+ const out: Record<string, number> = {}
+ for (const label of labels) out[label] = await refreshCount(label)
+ return out
+ }
+
+ async function expectOnlyIncremented(before: Record<string, number>, label: string) {
+ const after = await snapshot()
+ for (const [key, value] of Object.entries(before)) {
+ expect([key, after[key]]).toEqual([key, key === label ? value + 1 : value])
+ }
+ }
+
+ async function seedSharedHost(): Promise<string> {
+ const host = (await gateway.createResource(
+ 'hosts',
+ {
+ metadata: { name: SHARED_HOST },
+ spec: { host: SHARED_HOST, contextRef: 'ctx-a', secretRef: `${SHARED_HOST}-llm` },
+ },
+ SHARED_HOST_NS
+ )) as { metadata: { uid: string } }
+ return host.metadata.uid
+ }
+
+ function issueHccRefresh(hostUid: string): string {
+ return mcpHostJwt.issueMcpHostRefreshJwt(SHARED_HOST_NS, 'standalone', [SHARED_HOST], {
+ workflowControlScopes: ['workflow:list'],
+ hccCredential: { hostUid },
+ }).token
+ }
+
+ function postRefresh(token: string) {
+ return request(app)
+ .post('/api/v1/workflow-auth/refresh')
+ .set('Authorization', `Bearer ${token}`)
+ .send()
+ }
+
+ it('counts a transient Host lookup failure as host_unavailable', async () => {
+ const token = issueHccRefresh(await seedSharedHost())
+ gateway.failGetResource('hosts', { statusCode: 503 })
+ const before = await snapshot()
+
+ const res = await postRefresh(token)
+
+ expect(res.status).toBe(503)
+ expect(res.body).toEqual({ error: 'host_state_unavailable' })
+ await expectOnlyIncremented(before, 'host_unavailable')
+ })
+
+ it('counts a chain whose Host was recreated with a new uid as host_mismatch', async () => {
+ const oldUid = await seedSharedHost()
+ const token = issueHccRefresh(oldUid)
+ await gateway.deleteResource('hosts', SHARED_HOST, SHARED_HOST_NS)
+ expect(await seedSharedHost()).not.toBe(oldUid)
+ const before = await snapshot()
+
+ const res = await postRefresh(token)
+
+ expect(res.status).toBe(401)
+ await expectOnlyIncremented(before, 'host_mismatch')
+ })
+
+ it('counts a chain whose Host no longer exists as host_mismatch', async () => {
+ const hostUid = await seedSharedHost()
+ const token = issueHccRefresh(hostUid)
+ await gateway.deleteResource('hosts', SHARED_HOST, SHARED_HOST_NS)
+ const before = await snapshot()
+
+ const res = await postRefresh(token)
+
+ expect(res.status).toBe(401)
+ await expectOnlyIncremented(before, 'host_mismatch')
+ })
+
+ it('counts a hosts-namespace standalone chain without HCC audience as lineage_without_audience', async () => {
+ await seedSharedHost()
+ const token = mcpHostJwt.issueMcpHostRefreshJwt(
+ SHARED_HOST_NS,
+ 'standalone',
+ [SHARED_HOST],
+ { workflowControlScopes: ['workflow:list'] }
+ ).token
+ const before = await snapshot()
+
+ const res = await postRefresh(token)
+
+ expect(res.status).toBe(401)
+ await expectOnlyIncremented(before, 'lineage_without_audience')
+ })
+
+ it('counts a jti consumed between verify and consume as revoked', async () => {
+ const { refreshToken } = issueTestTokens()
+ conflictOnNextRefreshJtiInsert = true
+ const before = await snapshot()
+
+ const res = await postRefresh(refreshToken)
+
+ expect(res.status).toBe(401)
+ await expectOnlyIncremented(before, 'revoked')
+ })
+
+ it('keeps an invalid refresh token on the failed label', async () => {
+ const before = await snapshot()
+
+ const res = await postRefresh('invalid-token')
+
+ expect(res.status).toBe(401)
+ await expectOnlyIncremented(before, 'failed')
+ })
+ })
+
it('rejects access token used as refresh', async () => {
const { accessToken } = issueTestTokens()
const res = await request(app)