Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): republish the allowlist after a persisted refresh failure (#739)
Alfredo Lopez5ba6e677b028
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 54c78b6c1..4d0f51e90 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.620",+ "version": "0.1.621","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.620",+ "version": "0.1.621","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 2d941606a..f528c2ac0 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.620",+ "version": "0.1.621","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/app.ts b/control-api/src/app.tsindex 018e37b80..41cc4d1fe 100644--- a/control-api/src/app.ts+++ b/control-api/src/app.ts@@ -254,7 +254,7 @@ export function createApp(gateway: K8sGateway) {// gates the boundary; the public-facing cookie-authed endpoints live in// rpc-proxy and forward here with the user identity asserted in the body.api.use(createInternalOAuthRouter(gateway))- api.use(createInternalLlmProviderAttemptRoutes())+ api.use(createInternalLlmProviderAttemptRoutes(gateway.llmAllowedModelsConfigMap()))app.use('/api/v1', api)app.use((_req, res) => {diff --git a/control-api/src/routes/internal/llmProviderAttempts.ts b/control-api/src/routes/internal/llmProviderAttempts.tsindex 6862e067d..142c390bb 100644--- a/control-api/src/routes/internal/llmProviderAttempts.ts+++ b/control-api/src/routes/internal/llmProviderAttempts.ts@@ -8,14 +8,17 @@ import {} from '../../services/grokProviderAttemptFinalization.js'import {GrokProviderAttemptRedeemError,+ productionGrokRedeemDeps,redeemGrokProviderAttempt,} from '../../services/grokProviderAttemptRedemption.js'+import type { AllowedModelsConfigMapMaterializer } from '../../services/llmAllowedModelsConfigMap.js'import {LlmProviderAttemptFinalizeError,finalizeLlmProviderAttempt,} from '../../services/llmProviderAttemptFinalization.js'import {LlmProviderAttemptRedeemError,+ productionRedeemDeps,redeemLlmProviderAttempt,} from '../../services/llmProviderAttemptRedemption.js'import { isPlainObject } from '../../utils/isPlainObject.js'@@ -41,7 +44,9 @@ const FINALIZE_STATUS: Record<string, number> = {conflict: 409,}-export function createInternalLlmProviderAttemptRoutes(): Router {+export function createInternalLlmProviderAttemptRoutes(+ materializer: AllowedModelsConfigMapMaterializer+): Router {const router = Router()router.use('/internal/llm/provider-attempts', requireInternalService('codex-llm-proxy'))@@ -50,16 +55,19 @@ export function createInternalLlmProviderAttemptRoutes(): Router {asyncHandler(async (req, res) => {const body = isPlainObject(req.body) ? req.body : {}try {- const result = await redeemLlmProviderAttempt({- executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',- requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',- model: typeof body.model === 'string' ? body.model : undefined,- hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,- operation:- body.operation === 'completion_cancel' || body.operation === 'connection_test'- ? body.operation- : 'completion_stream',- })+ const result = await redeemLlmProviderAttempt(+ {+ executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',+ requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',+ model: typeof body.model === 'string' ? body.model : undefined,+ hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,+ operation:+ body.operation === 'completion_cancel' || body.operation === 'connection_test'+ ? body.operation+ : 'completion_stream',+ },+ productionRedeemDeps(materializer)+ )res.status(200).json(result)} catch (err) {if (err instanceof LlmProviderAttemptRedeemError) {@@ -100,16 +108,19 @@ export function createInternalLlmProviderAttemptRoutes(): Router {asyncHandler(async (req, res) => {const body = isPlainObject(req.body) ? req.body : {}try {- const result = await redeemGrokProviderAttempt({- executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',- requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',- model: typeof body.model === 'string' ? body.model : undefined,- hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,- operation:- body.operation === 'completion_cancel' || body.operation === 'connection_test'- ? body.operation- : 'completion_stream',- })+ const result = await redeemGrokProviderAttempt(+ {+ executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',+ requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',+ model: typeof body.model === 'string' ? body.model : undefined,+ hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,+ operation:+ body.operation === 'completion_cancel' || body.operation === 'connection_test'+ ? body.operation+ : 'completion_stream',+ },+ productionGrokRedeemDeps(materializer)+ )res.status(200).json(result)} catch (err) {if (err instanceof GrokProviderAttemptRedeemError) {diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.tsindex 7bc27b11e..c4ae09ce3 100644--- a/control-api/src/services/grokProviderAttemptRedemption.ts+++ b/control-api/src/services/grokProviderAttemptRedemption.ts@@ -7,12 +7,17 @@ import { config } from '../config.js'import { type DbClient, pool, withTransaction } from '../db.js'import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'import { rootLogger } from '../observability/logger.js'+import { llmAllowlistConfigMapWriteFailuresTotal } from '../observability/metrics.js'import { verifyGrokExecutionTicket } from './grokProviderAttemptTicket.js'import {getSafeGrokSubscriptionConnectionById,loadGrokSubscriptionSecrets,} from './grokSubscriptionConnection.js'import { GrokSubscriptionOAuthError, ensureFreshGrokAccessToken } from './grokSubscriptionOAuth.js'+import {+ type AllowedModelsConfigMapMaterializer,+ publishAllowedModelsConfigMapAfterGrantChange,+} from './llmAllowedModelsConfigMap.js'import { opaqueAttemptReceipt } from './llmProviderAttemptRedemption.js'import {loadLlmProviderAttempt,@@ -78,9 +83,13 @@ export type RedeemGrokAttemptDeps = {getConnectionById: typeof getSafeGrokSubscriptionConnectionByIdencryptionKey: BufferensureFreshAccessToken?: (connectionKey?: string) => Promise<void>+ /** Republishes the runtime allowlist ConfigMap after a grant change. */+ publishAllowlist: () => Promise<void>}-const defaultRedeemDeps = (): RedeemGrokAttemptDeps => ({+export const productionGrokRedeemDeps = (+ materializer: AllowedModelsConfigMapMaterializer+): RedeemGrokAttemptDeps => ({enabled: config.grokSubscriptionEnabled,db: pool,withTransaction,@@ -96,11 +105,35 @@ const defaultRedeemDeps = (): RedeemGrokAttemptDeps => ({enabled: config.grokSubscriptionEnabled,connectionKey: connectionKey ?? '',}),+ publishAllowlist: async () => {+ await publishAllowedModelsConfigMapAfterGrantChange(materializer)+ },})+/**+ * A refresh failure that persisted a connection status (a rejected refresh+ * token marks the row reauth_required) changed what the runtime ConfigMap+ * must say about the grant, so it is published before the redemption fails,+ * as the refresh and catalog-sync routes do. A failed publish is counted and+ * logged exactly as those routes count and log it; the redemption then+ * fails with its own code, because the proxy contract has no code for an+ * unwritten ConfigMap.+ */+async function publishAfterPersistedRefreshFailure(deps: RedeemGrokAttemptDeps): Promise<void> {+ try {+ await deps.publishAllowlist()+ } catch (err) {+ llmAllowlistConfigMapWriteFailuresTotal.inc({ phase: 'mutation' })+ log.error(+ { err, event: 'grok_allowed_models_cm_write_failed' },+ 'Grok refresh failure persisted a connection status but the runtime ConfigMap was not updated'+ )+ }+}+export async function redeemGrokProviderAttempt(input: RedeemGrokAttemptInput,- deps: RedeemGrokAttemptDeps = defaultRedeemDeps()+ deps: RedeemGrokAttemptDeps): Promise<RedeemGrokAttemptSuccess> {if (!deps.enabled) {throw new GrokProviderAttemptRedeemError('disabled', 'Grok subscription is disabled')@@ -161,6 +194,9 @@ export async function redeemGrokProviderAttempt(await deps.ensureFreshAccessToken(assigned.connectionKey)} catch (err) {if (err instanceof GrokSubscriptionOAuthError) {+ if (err.persistedConnectionStatus === true) {+ await publishAfterPersistedRefreshFailure(deps)+ }throw new GrokProviderAttemptRedeemError(err.code === 'no_grant' || err.code === 'not_connected' || err.code === 'reauth_required'? 'no_grant'diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.tsindex c23b8fa21..5c8d451ad 100644--- a/control-api/src/services/llmProviderAttemptRedemption.ts+++ b/control-api/src/services/llmProviderAttemptRedemption.ts@@ -3,6 +3,7 @@ import { config } from '../config.js'import { type DbClient, pool, withTransaction } from '../db.js'import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'import { rootLogger } from '../observability/logger.js'+import { llmAllowlistConfigMapWriteFailuresTotal } from '../observability/metrics.js'import { chatgptAccountIdFromJwt } from './chatgptAccountId.js'import {getSafeCodexSubscriptionConnectionById,@@ -12,6 +13,10 @@ import {CodexSubscriptionOAuthError,ensureFreshCodexAccessToken,} from './codexSubscriptionOAuth.js'+import {+ type AllowedModelsConfigMapMaterializer,+ publishAllowedModelsConfigMapAfterGrantChange,+} from './llmAllowedModelsConfigMap.js'import {loadLlmProviderAttempt,lockLlmProviderAttemptTicket,@@ -88,9 +93,13 @@ export type RedeemAttemptDeps = {getConnectionById: typeof getSafeCodexSubscriptionConnectionByIdencryptionKey: BufferensureFreshAccessToken?: (connectionKey?: string) => Promise<void>+ /** Republishes the runtime allowlist ConfigMap after a grant change. */+ publishAllowlist: () => Promise<void>}-const defaultRedeemDeps = (): RedeemAttemptDeps => ({+export const productionRedeemDeps = (+ materializer: AllowedModelsConfigMapMaterializer+): RedeemAttemptDeps => ({enabled: config.codexSubscriptionEnabled,db: pool,withTransaction,@@ -107,11 +116,34 @@ const defaultRedeemDeps = (): RedeemAttemptDeps => ({enabled: config.codexSubscriptionEnabled,connectionKey,}),+ publishAllowlist: async () => {+ await publishAllowedModelsConfigMapAfterGrantChange(materializer)+ },})+/**+ * A refresh failure that persisted a connection status (a rejected refresh+ * token marks the row reauth_required) changed what the runtime ConfigMap+ * must say about the grant, so it is published before the redemption fails,+ * as the refresh and catalog-sync routes do. A failed publish is counted and+ * logged exactly as the refresh route counts and logs it; the redemption+ * then fails with its own code, which already answers 503 to the proxy.+ */+async function publishAfterPersistedRefreshFailure(deps: RedeemAttemptDeps): Promise<void> {+ try {+ await deps.publishAllowlist()+ } catch (err) {+ llmAllowlistConfigMapWriteFailuresTotal.inc({ phase: 'mutation' })+ log.error(+ { err, event: 'codex_allowed_models_cm_write_failed' },+ 'Codex refresh failure persisted a connection status but the runtime ConfigMap was not updated'+ )+ }+}+export async function redeemLlmProviderAttempt(input: RedeemAttemptInput,- deps: RedeemAttemptDeps = defaultRedeemDeps()+ deps: RedeemAttemptDeps): Promise<RedeemAttemptSuccess> {if (!deps.enabled) {throw new LlmProviderAttemptRedeemError('disabled', 'Codex subscription is disabled')@@ -143,6 +175,9 @@ export async function redeemLlmProviderAttempt(await deps.ensureFreshAccessToken(refreshKey)} catch (err) {if (err instanceof CodexSubscriptionOAuthError) {+ if (err.persistedConnectionStatus === true) {+ await publishAfterPersistedRefreshFailure(deps)+ }throw new LlmProviderAttemptRedeemError(err.code === 'no_grant' || err.code === 'not_connected'? 'no_grant'