Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(control-api): require redemption to republish the allowlist (#739)

Alfredo Lopez18f1c8b8c73e
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index e82d0cdbe..54c78b6c1 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.619",
+ "version": "0.1.620",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.619",
+ "version": "0.1.620",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 39a202743..2d941606a 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.619",
+ "version": "0.1.620",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts b/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts
index 461f15f01..c58fbd41f 100644
--- a/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts
+++ b/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts
@@ -42,6 +42,9 @@ const { createInternalLlmProviderAttemptRoutes } =
const REDEEM = '/internal/llm/grok/provider-attempts/redeem'
const FINALIZE = '/internal/llm/grok/provider-attempts/finalize'
+/** The ConfigMap writer the gateway hands the routes; the cluster is outside this process. */
+const materializer = { materialize: vi.fn(async () => {}) }
+
/** Stand-in for internalServiceAuth: the header names the authenticated caller. */
function makeApp() {
const app = express()
@@ -55,7 +58,7 @@ function makeApp() {
}
next()
})
- app.use(createInternalLlmProviderAttemptRoutes())
+ app.use(createInternalLlmProviderAttemptRoutes(materializer))
app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => {
res.status(500).json({ error: 'internal', name: (err as Error)?.name })
})
@@ -65,6 +68,7 @@ function makeApp() {
describe('internal Grok provider-attempt routes', () => {
beforeEach(() => {
for (const fn of Object.values(services)) fn.mockReset()
+ materializer.materialize.mockClear()
})
describe('service identity', () => {
@@ -105,16 +109,40 @@ describe('internal Grok provider-attempt routes', () => {
operation: 'drop_tables',
})
expect(res.status).toBe(200)
- expect(services.redeemGrok).toHaveBeenCalledWith({
- executionTicket: 'ticket',
- requestHash: 'a'.repeat(64),
- model: 'grok-4.6',
- hostRef: undefined,
- operation: 'completion_stream',
- })
+ expect(services.redeemGrok).toHaveBeenCalledWith(
+ {
+ executionTicket: 'ticket',
+ requestHash: 'a'.repeat(64),
+ model: 'grok-4.6',
+ hostRef: undefined,
+ operation: 'completion_stream',
+ },
+ expect.objectContaining({ publishAllowlist: expect.any(Function) })
+ )
expect(services.redeemCodex).not.toHaveBeenCalled()
})
+ it.each([
+ ['grok-llm-proxy', REDEEM, services.redeemGrok],
+ ['codex-llm-proxy', '/internal/llm/provider-attempts/redeem', services.redeemCodex],
+ ] as const)(
+ 'R9-19 hands the %s redemption a publish bound to the gateway ConfigMap writer',
+ async (service, path, redeem) => {
+ redeem.mockResolvedValue({ accessToken: 'access', attemptReceipt: 'r' })
+ const res = await request(makeApp())
+ .post(path)
+ .set('x-test-service', service)
+ .send({ executionTicket: 'ticket', requestHash: 'h' })
+ expect(res.status).toBe(200)
+ expect(redeem).toHaveBeenCalledTimes(1)
+
+ const deps = redeem.mock.calls[0]?.[1] as { publishAllowlist: () => Promise<void> }
+ expect(materializer.materialize).not.toHaveBeenCalled()
+ await deps.publishAllowlist()
+ expect(materializer.materialize).toHaveBeenCalledTimes(1)
+ }
+ )
+
const redeemStatus: Array<[GrokProviderAttemptRedeemErrorCode, number]> = [
['disabled', 404],
['ticket_invalid', 403],
diff --git a/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts b/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts
index f9eb643d9..0e060a87e 100644
--- a/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts
+++ b/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts
@@ -169,7 +169,10 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreS
})
}
- function redeemDeps(ensureFreshAccessToken: (connectionKey?: string) => Promise<void>) {
+ function redeemDeps(
+ ensureFreshAccessToken: (connectionKey?: string) => Promise<void>,
+ publishAllowlist: () => Promise<void> = vi.fn(async () => {})
+ ) {
return {
enabled: true as const,
db: pool,
@@ -177,6 +180,7 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreS
loadSecrets: loadGrokSubscriptionSecrets,
getConnectionById: getSafeGrokSubscriptionConnectionById,
ensureFreshAccessToken,
+ publishAllowlist,
withTransaction: (async <T>(work: (tx: never) => Promise<T>): Promise<T> => {
const client = await pool.connect()
try {
@@ -259,15 +263,19 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreS
})
const { attempt, issued } = await issueTicket(connection)
const fetchFn = vi.fn(async () => jsonResponse(400, { error: 'invalid_grant' }))
+ const publishAllowlist = vi.fn(async () => {})
await expect(
redeemGrokProviderAttempt(
{ executionTicket: issued.executionTicket, requestHash: attempt.requestHash },
- redeemDeps(realEnsureFresh(fetchFn as unknown as typeof fetch))
+ redeemDeps(realEnsureFresh(fetchFn as unknown as typeof fetch), publishAllowlist)
)
).rejects.toMatchObject({ name: 'GrokProviderAttemptRedeemError', code: 'no_grant' })
expect(fetchFn).toHaveBeenCalledTimes(1)
+ // R9-19: the row below is now reauth_required, so the ConfigMap is owed a
+ // republish before the redemption fails.
+ expect(publishAllowlist).toHaveBeenCalledTimes(1)
expect(await ledgerState(issued.claims.jti, attempt.id)).toEqual({
ticketStatus: 'issued',
redeemedAt: null,
diff --git a/control-api/test/services.grokProviderAttemptRedemption.test.ts b/control-api/test/services.grokProviderAttemptRedemption.test.ts
index e21c48ce7..5a2423407 100644
--- a/control-api/test/services.grokProviderAttemptRedemption.test.ts
+++ b/control-api/test/services.grokProviderAttemptRedemption.test.ts
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { config } from '../src/config.js'
+import { llmAllowlistConfigMapWriteFailuresTotal } from '../src/observability/metrics.js'
import {
GrokProviderAttemptRedeemError,
redeemGrokProviderAttempt,
@@ -212,6 +213,7 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {
connection?: unknown
markRedeemed?: boolean
secrets?: unknown
+ publishAllowlist?: () => Promise<void>
}
function arrange(scenario: Scenario = {}) {
@@ -254,6 +256,7 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {
.mockResolvedValue('assigned' in scenario ? scenario.assigned : liveConnection())
const tx = { query: vi.fn() }
const withTransaction = vi.fn(async (work: (db: typeof tx) => unknown) => work(tx))
+ const publishAllowlist = vi.fn(scenario.publishAllowlist ?? (async () => {}))
const run = () =>
redeemGrokProviderAttempt(
{
@@ -268,11 +271,13 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {
loadSecrets,
getConnectionById,
encryptionKey: Buffer.alloc(32),
+ publishAllowlist,
...(ensureFresh ? { ensureFreshAccessToken: ensureFresh } : {}),
}
)
return {
run,
+ publishAllowlist,
peek,
lock,
connectionById,
@@ -383,6 +388,73 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {
expect(h.markRedeemed).not.toHaveBeenCalled()
})
+ /**
+ * R9-19: a refresh that wrote the connection row changed what the allowlist
+ * ConfigMap must say, and mcp-host and HCC read the ConfigMap, not Postgres.
+ * Redemption republishes before it fails, as the refresh route does.
+ */
+ function persistedRejection() {
+ return async () => {
+ throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
+ persistedConnectionStatus: true,
+ })
+ }
+ }
+
+ async function mutationWriteFailures(): Promise<number> {
+ const metric = await llmAllowlistConfigMapWriteFailuresTotal.get()
+ return metric.values.find(v => v.labels.phase === 'mutation')?.value ?? 0
+ }
+
+ it('R9-19 republishes the allowlist before failing when the refresh persisted a status', async () => {
+ const h = arrange({ ensureFresh: persistedRejection() })
+
+ await expect(h.run()).rejects.toMatchObject({
+ name: 'GrokProviderAttemptRedeemError',
+ code: 'no_grant',
+ })
+
+ expect(h.ensureFresh).toHaveBeenCalledWith('team-grok')
+ expect(h.publishAllowlist).toHaveBeenCalledTimes(1)
+ expect(h.withTransaction).not.toHaveBeenCalled()
+ })
+
+ it('R9-19 does not republish when the refresh failed without writing the row', async () => {
+ const h = arrange({
+ ensureFresh: async () => {
+ throw new GrokSubscriptionOAuthError('provider_unavailable', 'refresh failed')
+ },
+ })
+
+ await expect(h.run()).rejects.toMatchObject({ code: 'provider_unavailable' })
+
+ // Liveness witness: the refresh ran and threw, so the missing publish is a
+ // decision and not a path that never reached the catch.
+ expect(h.ensureFresh).toHaveBeenCalledTimes(1)
+ await expect(h.ensureFresh?.mock.results[0]?.value as Promise<void>).rejects.toBeInstanceOf(
+ GrokSubscriptionOAuthError
+ )
+ expect(h.publishAllowlist).not.toHaveBeenCalled()
+ })
+
+ it('R9-19 counts a failed republish and still fails the redemption with its own code', async () => {
+ const before = await mutationWriteFailures()
+ const h = arrange({
+ ensureFresh: persistedRejection(),
+ publishAllowlist: async () => {
+ throw new Error('configmap write refused')
+ },
+ })
+
+ await expect(h.run()).rejects.toMatchObject({
+ name: 'GrokProviderAttemptRedeemError',
+ code: 'no_grant',
+ })
+
+ expect(h.publishAllowlist).toHaveBeenCalledTimes(1)
+ expect(await mutationWriteFailures()).toBe(before + 1)
+ })
+
it('rethrows a non-OAuth refresh failure unchanged', async () => {
const boom = new Error('socket hang up')
const h = arrange({
diff --git a/control-api/test/services.llmProviderAttemptRedemption.test.ts b/control-api/test/services.llmProviderAttemptRedemption.test.ts
index a303adcdb..087906096 100644
--- a/control-api/test/services.llmProviderAttemptRedemption.test.ts
+++ b/control-api/test/services.llmProviderAttemptRedemption.test.ts
@@ -1,6 +1,8 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { config } from '../src/config.js'
+import { llmAllowlistConfigMapWriteFailuresTotal } from '../src/observability/metrics.js'
import * as connection from '../src/services/codexSubscriptionConnection.js'
+import { CodexSubscriptionOAuthError } from '../src/services/codexSubscriptionOAuth.js'
import {
LlmProviderAttemptRedeemError,
redeemLlmProviderAttempt,
@@ -171,8 +173,24 @@ describe('redeemLlmProviderAttempt', () => {
it('rejects a requestHash mismatch', async () => {
await expect(
- redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: 'c'.repeat(64) })
- ).rejects.toBeInstanceOf(LlmProviderAttemptRedeemError)
+ redeemLlmProviderAttempt(
+ { executionTicket: 'ticket', requestHash: 'c'.repeat(64) },
+ {
+ enabled: true,
+ db: { query: vi.fn() },
+ getConnectionById: vi.fn(),
+ withTransaction: async () => {
+ throw new Error('should not run')
+ },
+ loadSecrets: async () => null,
+ encryptionKey: Buffer.alloc(32),
+ publishAllowlist: vi.fn(),
+ }
+ )
+ ).rejects.toMatchObject({
+ name: 'LlmProviderAttemptRedeemError',
+ code: 'request_hash_mismatch',
+ })
})
it('refuses to redeem a pre-revoke ticket after the grant is revoked', async () => {
@@ -278,4 +296,107 @@ describe('redeemLlmProviderAttempt', () => {
)
).rejects.toMatchObject({ code: 'connection_unavailable' })
})
+
+ /**
+ * R9-19: a refresh that wrote the connection row (a rejected refresh token
+ * moves it to `reauth_required`) changed what the allowlist ConfigMap must
+ * say. mcp-host and HCC read the ConfigMap, not Postgres, so redemption
+ * republishes before it fails, as the refresh and catalog-sync routes do.
+ */
+ function refreshingDeps(
+ ensureFreshAccessToken: (connectionKey?: string) => Promise<void>,
+ publishAllowlist: () => Promise<void>
+ ) {
+ vi.mocked(ticket.verifyCodexExecutionTicket).mockReturnValue({
+ ...CLAIMS,
+ connectionId: CONNECTION_ID,
+ })
+ return {
+ enabled: true,
+ db: { query: vi.fn() },
+ getConnectionById: vi.fn(async () => SAFE_CONNECTION),
+ withTransaction: vi.fn(async () => {
+ throw new Error('should not open the redemption transaction')
+ }),
+ loadSecrets: vi.fn(async () => null),
+ encryptionKey: Buffer.alloc(32),
+ ensureFreshAccessToken: vi.fn(ensureFreshAccessToken),
+ publishAllowlist: vi.fn(publishAllowlist),
+ }
+ }
+
+ async function mutationWriteFailures(): Promise<number> {
+ const metric = await llmAllowlistConfigMapWriteFailuresTotal.get()
+ return metric.values.find(v => v.labels.phase === 'mutation')?.value ?? 0
+ }
+
+ it('R9-19 republishes the allowlist before failing when the refresh persisted a status', async () => {
+ const deps = refreshingDeps(
+ async () => {
+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
+ persistedConnectionStatus: true,
+ })
+ },
+ async () => {}
+ )
+
+ await expect(
+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)
+ ).rejects.toMatchObject({
+ name: 'LlmProviderAttemptRedeemError',
+ code: 'connection_unavailable',
+ })
+
+ expect(deps.ensureFreshAccessToken).toHaveBeenCalledWith('team-plus')
+ expect(deps.publishAllowlist).toHaveBeenCalledTimes(1)
+ expect(deps.withTransaction).not.toHaveBeenCalled()
+ })
+
+ it('R9-19 does not republish when the refresh failed without writing the row', async () => {
+ const deps = refreshingDeps(
+ async () => {
+ throw new CodexSubscriptionOAuthError(
+ 'provider_unavailable',
+ 'refresh token exchange failed'
+ )
+ },
+ async () => {}
+ )
+
+ await expect(
+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)
+ ).rejects.toBeInstanceOf(LlmProviderAttemptRedeemError)
+
+ // Liveness witness: the refresh ran and threw, so the missing publish is a
+ // decision and not a path that never reached the catch.
+ expect(deps.ensureFreshAccessToken).toHaveBeenCalledTimes(1)
+ await expect(
+ deps.ensureFreshAccessToken.mock.results[0]?.value as Promise<void>
+ ).rejects.toBeInstanceOf(CodexSubscriptionOAuthError)
+ expect(deps.publishAllowlist).not.toHaveBeenCalled()
+ })
+
+ it('R9-19 counts a failed republish and still fails the redemption with its own code', async () => {
+ const before = await mutationWriteFailures()
+ const deps = refreshingDeps(
+ async () => {
+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
+ persistedConnectionStatus: true,
+ })
+ },
+ async () => {
+ throw new Error('configmap write refused')
+ }
+ )
+
+ await expect(
+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)
+ ).rejects.toMatchObject({
+ name: 'LlmProviderAttemptRedeemError',
+ code: 'connection_unavailable',
+ })
+
+ expect(deps.publishAllowlist).toHaveBeenCalledTimes(1)
+ expect(await mutationWriteFailures()).toBe(before + 1)
+ })
})