Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(control-api): require redemption to republish the allowlist (#739)
Alfredo Lopez18f1c8b8c73e
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex e82d0cdbe..54c78b6c1 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.619",+ "version": "0.1.620","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.619",+ "version": "0.1.620","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 39a202743..2d941606a 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.619",+ "version": "0.1.620","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts b/control-api/test/routes.internal.llmProviderAttempts.grok.test.tsindex 461f15f01..c58fbd41f 100644--- a/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts+++ b/control-api/test/routes.internal.llmProviderAttempts.grok.test.ts@@ -42,6 +42,9 @@ const { createInternalLlmProviderAttemptRoutes } =const REDEEM = '/internal/llm/grok/provider-attempts/redeem'const FINALIZE = '/internal/llm/grok/provider-attempts/finalize'+/** The ConfigMap writer the gateway hands the routes; the cluster is outside this process. */+const materializer = { materialize: vi.fn(async () => {}) }+/** Stand-in for internalServiceAuth: the header names the authenticated caller. */function makeApp() {const app = express()@@ -55,7 +58,7 @@ function makeApp() {}next()})- app.use(createInternalLlmProviderAttemptRoutes())+ app.use(createInternalLlmProviderAttemptRoutes(materializer))app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => {res.status(500).json({ error: 'internal', name: (err as Error)?.name })})@@ -65,6 +68,7 @@ function makeApp() {describe('internal Grok provider-attempt routes', () => {beforeEach(() => {for (const fn of Object.values(services)) fn.mockReset()+ materializer.materialize.mockClear()})describe('service identity', () => {@@ -105,16 +109,40 @@ describe('internal Grok provider-attempt routes', () => {operation: 'drop_tables',})expect(res.status).toBe(200)- expect(services.redeemGrok).toHaveBeenCalledWith({- executionTicket: 'ticket',- requestHash: 'a'.repeat(64),- model: 'grok-4.6',- hostRef: undefined,- operation: 'completion_stream',- })+ expect(services.redeemGrok).toHaveBeenCalledWith(+ {+ executionTicket: 'ticket',+ requestHash: 'a'.repeat(64),+ model: 'grok-4.6',+ hostRef: undefined,+ operation: 'completion_stream',+ },+ expect.objectContaining({ publishAllowlist: expect.any(Function) })+ )expect(services.redeemCodex).not.toHaveBeenCalled()})+ it.each([+ ['grok-llm-proxy', REDEEM, services.redeemGrok],+ ['codex-llm-proxy', '/internal/llm/provider-attempts/redeem', services.redeemCodex],+ ] as const)(+ 'R9-19 hands the %s redemption a publish bound to the gateway ConfigMap writer',+ async (service, path, redeem) => {+ redeem.mockResolvedValue({ accessToken: 'access', attemptReceipt: 'r' })+ const res = await request(makeApp())+ .post(path)+ .set('x-test-service', service)+ .send({ executionTicket: 'ticket', requestHash: 'h' })+ expect(res.status).toBe(200)+ expect(redeem).toHaveBeenCalledTimes(1)++ const deps = redeem.mock.calls[0]?.[1] as { publishAllowlist: () => Promise<void> }+ expect(materializer.materialize).not.toHaveBeenCalled()+ await deps.publishAllowlist()+ expect(materializer.materialize).toHaveBeenCalledTimes(1)+ }+ )+const redeemStatus: Array<[GrokProviderAttemptRedeemErrorCode, number]> = [['disabled', 404],['ticket_invalid', 403],diff --git a/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts b/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.tsindex f9eb643d9..0e060a87e 100644--- a/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts+++ b/control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts@@ -169,7 +169,10 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreS})}- function redeemDeps(ensureFreshAccessToken: (connectionKey?: string) => Promise<void>) {+ function redeemDeps(+ ensureFreshAccessToken: (connectionKey?: string) => Promise<void>,+ publishAllowlist: () => Promise<void> = vi.fn(async () => {})+ ) {return {enabled: true as const,db: pool,@@ -177,6 +180,7 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreSloadSecrets: loadGrokSubscriptionSecrets,getConnectionById: getSafeGrokSubscriptionConnectionById,ensureFreshAccessToken,+ publishAllowlist,withTransaction: (async <T>(work: (tx: never) => Promise<T>): Promise<T> => {const client = await pool.connect()try {@@ -259,15 +263,19 @@ describeRealPostgres('Grok ticket redemption with token refresh on real PostgreS})const { attempt, issued } = await issueTicket(connection)const fetchFn = vi.fn(async () => jsonResponse(400, { error: 'invalid_grant' }))+ const publishAllowlist = vi.fn(async () => {})await expect(redeemGrokProviderAttempt({ executionTicket: issued.executionTicket, requestHash: attempt.requestHash },- redeemDeps(realEnsureFresh(fetchFn as unknown as typeof fetch))+ redeemDeps(realEnsureFresh(fetchFn as unknown as typeof fetch), publishAllowlist))).rejects.toMatchObject({ name: 'GrokProviderAttemptRedeemError', code: 'no_grant' })expect(fetchFn).toHaveBeenCalledTimes(1)+ // R9-19: the row below is now reauth_required, so the ConfigMap is owed a+ // republish before the redemption fails.+ expect(publishAllowlist).toHaveBeenCalledTimes(1)expect(await ledgerState(issued.claims.jti, attempt.id)).toEqual({ticketStatus: 'issued',redeemedAt: null,diff --git a/control-api/test/services.grokProviderAttemptRedemption.test.ts b/control-api/test/services.grokProviderAttemptRedemption.test.tsindex e21c48ce7..5a2423407 100644--- a/control-api/test/services.grokProviderAttemptRedemption.test.ts+++ b/control-api/test/services.grokProviderAttemptRedemption.test.ts@@ -1,5 +1,6 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { config } from '../src/config.js'+import { llmAllowlistConfigMapWriteFailuresTotal } from '../src/observability/metrics.js'import {GrokProviderAttemptRedeemError,redeemGrokProviderAttempt,@@ -212,6 +213,7 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {connection?: unknownmarkRedeemed?: booleansecrets?: unknown+ publishAllowlist?: () => Promise<void>}function arrange(scenario: Scenario = {}) {@@ -254,6 +256,7 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {.mockResolvedValue('assigned' in scenario ? scenario.assigned : liveConnection())const tx = { query: vi.fn() }const withTransaction = vi.fn(async (work: (db: typeof tx) => unknown) => work(tx))+ const publishAllowlist = vi.fn(scenario.publishAllowlist ?? (async () => {}))const run = () =>redeemGrokProviderAttempt({@@ -268,11 +271,13 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {loadSecrets,getConnectionById,encryptionKey: Buffer.alloc(32),+ publishAllowlist,...(ensureFresh ? { ensureFreshAccessToken: ensureFresh } : {}),})return {run,+ publishAllowlist,peek,lock,connectionById,@@ -383,6 +388,73 @@ describe('redeemGrokProviderAttempt error-path matrix', () => {expect(h.markRedeemed).not.toHaveBeenCalled()})+ /**+ * R9-19: a refresh that wrote the connection row changed what the allowlist+ * ConfigMap must say, and mcp-host and HCC read the ConfigMap, not Postgres.+ * Redemption republishes before it fails, as the refresh route does.+ */+ function persistedRejection() {+ return async () => {+ throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {+ persistedConnectionStatus: true,+ })+ }+ }++ async function mutationWriteFailures(): Promise<number> {+ const metric = await llmAllowlistConfigMapWriteFailuresTotal.get()+ return metric.values.find(v => v.labels.phase === 'mutation')?.value ?? 0+ }++ it('R9-19 republishes the allowlist before failing when the refresh persisted a status', async () => {+ const h = arrange({ ensureFresh: persistedRejection() })++ await expect(h.run()).rejects.toMatchObject({+ name: 'GrokProviderAttemptRedeemError',+ code: 'no_grant',+ })++ expect(h.ensureFresh).toHaveBeenCalledWith('team-grok')+ expect(h.publishAllowlist).toHaveBeenCalledTimes(1)+ expect(h.withTransaction).not.toHaveBeenCalled()+ })++ it('R9-19 does not republish when the refresh failed without writing the row', async () => {+ const h = arrange({+ ensureFresh: async () => {+ throw new GrokSubscriptionOAuthError('provider_unavailable', 'refresh failed')+ },+ })++ await expect(h.run()).rejects.toMatchObject({ code: 'provider_unavailable' })++ // Liveness witness: the refresh ran and threw, so the missing publish is a+ // decision and not a path that never reached the catch.+ expect(h.ensureFresh).toHaveBeenCalledTimes(1)+ await expect(h.ensureFresh?.mock.results[0]?.value as Promise<void>).rejects.toBeInstanceOf(+ GrokSubscriptionOAuthError+ )+ expect(h.publishAllowlist).not.toHaveBeenCalled()+ })++ it('R9-19 counts a failed republish and still fails the redemption with its own code', async () => {+ const before = await mutationWriteFailures()+ const h = arrange({+ ensureFresh: persistedRejection(),+ publishAllowlist: async () => {+ throw new Error('configmap write refused')+ },+ })++ await expect(h.run()).rejects.toMatchObject({+ name: 'GrokProviderAttemptRedeemError',+ code: 'no_grant',+ })++ expect(h.publishAllowlist).toHaveBeenCalledTimes(1)+ expect(await mutationWriteFailures()).toBe(before + 1)+ })+it('rethrows a non-OAuth refresh failure unchanged', async () => {const boom = new Error('socket hang up')const h = arrange({diff --git a/control-api/test/services.llmProviderAttemptRedemption.test.ts b/control-api/test/services.llmProviderAttemptRedemption.test.tsindex a303adcdb..087906096 100644--- a/control-api/test/services.llmProviderAttemptRedemption.test.ts+++ b/control-api/test/services.llmProviderAttemptRedemption.test.ts@@ -1,6 +1,8 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import { config } from '../src/config.js'+import { llmAllowlistConfigMapWriteFailuresTotal } from '../src/observability/metrics.js'import * as connection from '../src/services/codexSubscriptionConnection.js'+import { CodexSubscriptionOAuthError } from '../src/services/codexSubscriptionOAuth.js'import {LlmProviderAttemptRedeemError,redeemLlmProviderAttempt,@@ -171,8 +173,24 @@ describe('redeemLlmProviderAttempt', () => {it('rejects a requestHash mismatch', async () => {await expect(- redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: 'c'.repeat(64) })- ).rejects.toBeInstanceOf(LlmProviderAttemptRedeemError)+ redeemLlmProviderAttempt(+ { executionTicket: 'ticket', requestHash: 'c'.repeat(64) },+ {+ enabled: true,+ db: { query: vi.fn() },+ getConnectionById: vi.fn(),+ withTransaction: async () => {+ throw new Error('should not run')+ },+ loadSecrets: async () => null,+ encryptionKey: Buffer.alloc(32),+ publishAllowlist: vi.fn(),+ }+ )+ ).rejects.toMatchObject({+ name: 'LlmProviderAttemptRedeemError',+ code: 'request_hash_mismatch',+ })})it('refuses to redeem a pre-revoke ticket after the grant is revoked', async () => {@@ -278,4 +296,107 @@ describe('redeemLlmProviderAttempt', () => {)).rejects.toMatchObject({ code: 'connection_unavailable' })})++ /**+ * R9-19: a refresh that wrote the connection row (a rejected refresh token+ * moves it to `reauth_required`) changed what the allowlist ConfigMap must+ * say. mcp-host and HCC read the ConfigMap, not Postgres, so redemption+ * republishes before it fails, as the refresh and catalog-sync routes do.+ */+ function refreshingDeps(+ ensureFreshAccessToken: (connectionKey?: string) => Promise<void>,+ publishAllowlist: () => Promise<void>+ ) {+ vi.mocked(ticket.verifyCodexExecutionTicket).mockReturnValue({+ ...CLAIMS,+ connectionId: CONNECTION_ID,+ })+ return {+ enabled: true,+ db: { query: vi.fn() },+ getConnectionById: vi.fn(async () => SAFE_CONNECTION),+ withTransaction: vi.fn(async () => {+ throw new Error('should not open the redemption transaction')+ }),+ loadSecrets: vi.fn(async () => null),+ encryptionKey: Buffer.alloc(32),+ ensureFreshAccessToken: vi.fn(ensureFreshAccessToken),+ publishAllowlist: vi.fn(publishAllowlist),+ }+ }++ async function mutationWriteFailures(): Promise<number> {+ const metric = await llmAllowlistConfigMapWriteFailuresTotal.get()+ return metric.values.find(v => v.labels.phase === 'mutation')?.value ?? 0+ }++ it('R9-19 republishes the allowlist before failing when the refresh persisted a status', async () => {+ const deps = refreshingDeps(+ async () => {+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {+ persistedConnectionStatus: true,+ })+ },+ async () => {}+ )++ await expect(+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)+ ).rejects.toMatchObject({+ name: 'LlmProviderAttemptRedeemError',+ code: 'connection_unavailable',+ })++ expect(deps.ensureFreshAccessToken).toHaveBeenCalledWith('team-plus')+ expect(deps.publishAllowlist).toHaveBeenCalledTimes(1)+ expect(deps.withTransaction).not.toHaveBeenCalled()+ })++ it('R9-19 does not republish when the refresh failed without writing the row', async () => {+ const deps = refreshingDeps(+ async () => {+ throw new CodexSubscriptionOAuthError(+ 'provider_unavailable',+ 'refresh token exchange failed'+ )+ },+ async () => {}+ )++ await expect(+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)+ ).rejects.toBeInstanceOf(LlmProviderAttemptRedeemError)++ // Liveness witness: the refresh ran and threw, so the missing publish is a+ // decision and not a path that never reached the catch.+ expect(deps.ensureFreshAccessToken).toHaveBeenCalledTimes(1)+ await expect(+ deps.ensureFreshAccessToken.mock.results[0]?.value as Promise<void>+ ).rejects.toBeInstanceOf(CodexSubscriptionOAuthError)+ expect(deps.publishAllowlist).not.toHaveBeenCalled()+ })++ it('R9-19 counts a failed republish and still fails the redemption with its own code', async () => {+ const before = await mutationWriteFailures()+ const deps = refreshingDeps(+ async () => {+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {+ persistedConnectionStatus: true,+ })+ },+ async () => {+ throw new Error('configmap write refused')+ }+ )++ await expect(+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: CLAIMS.requestHash }, deps)+ ).rejects.toMatchObject({+ name: 'LlmProviderAttemptRedeemError',+ code: 'connection_unavailable',+ })++ expect(deps.publishAllowlist).toHaveBeenCalledTimes(1)+ expect(await mutationWriteFailures()).toBe(before + 1)+ })})