Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): republish the allowlist after a persisted refresh failure (#739)

Alfredo Lopez5ba6e677b028
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 54c78b6c1..4d0f51e90 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.620",
+ "version": "0.1.621",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.620",
+ "version": "0.1.621",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 2d941606a..f528c2ac0 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.620",
+ "version": "0.1.621",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/app.ts b/control-api/src/app.ts
index 018e37b80..41cc4d1fe 100644
--- a/control-api/src/app.ts
+++ b/control-api/src/app.ts
@@ -254,7 +254,7 @@ export function createApp(gateway: K8sGateway) {
// gates the boundary; the public-facing cookie-authed endpoints live in
// rpc-proxy and forward here with the user identity asserted in the body.
api.use(createInternalOAuthRouter(gateway))
- api.use(createInternalLlmProviderAttemptRoutes())
+ api.use(createInternalLlmProviderAttemptRoutes(gateway.llmAllowedModelsConfigMap()))
app.use('/api/v1', api)
app.use((_req, res) => {
diff --git a/control-api/src/routes/internal/llmProviderAttempts.ts b/control-api/src/routes/internal/llmProviderAttempts.ts
index 6862e067d..142c390bb 100644
--- a/control-api/src/routes/internal/llmProviderAttempts.ts
+++ b/control-api/src/routes/internal/llmProviderAttempts.ts
@@ -8,14 +8,17 @@ import {
} from '../../services/grokProviderAttemptFinalization.js'
import {
GrokProviderAttemptRedeemError,
+ productionGrokRedeemDeps,
redeemGrokProviderAttempt,
} from '../../services/grokProviderAttemptRedemption.js'
+import type { AllowedModelsConfigMapMaterializer } from '../../services/llmAllowedModelsConfigMap.js'
import {
LlmProviderAttemptFinalizeError,
finalizeLlmProviderAttempt,
} from '../../services/llmProviderAttemptFinalization.js'
import {
LlmProviderAttemptRedeemError,
+ productionRedeemDeps,
redeemLlmProviderAttempt,
} from '../../services/llmProviderAttemptRedemption.js'
import { isPlainObject } from '../../utils/isPlainObject.js'
@@ -41,7 +44,9 @@ const FINALIZE_STATUS: Record<string, number> = {
conflict: 409,
}
-export function createInternalLlmProviderAttemptRoutes(): Router {
+export function createInternalLlmProviderAttemptRoutes(
+ materializer: AllowedModelsConfigMapMaterializer
+): Router {
const router = Router()
router.use('/internal/llm/provider-attempts', requireInternalService('codex-llm-proxy'))
@@ -50,16 +55,19 @@ export function createInternalLlmProviderAttemptRoutes(): Router {
asyncHandler(async (req, res) => {
const body = isPlainObject(req.body) ? req.body : {}
try {
- const result = await redeemLlmProviderAttempt({
- executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',
- requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',
- model: typeof body.model === 'string' ? body.model : undefined,
- hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,
- operation:
- body.operation === 'completion_cancel' || body.operation === 'connection_test'
- ? body.operation
- : 'completion_stream',
- })
+ const result = await redeemLlmProviderAttempt(
+ {
+ executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',
+ requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',
+ model: typeof body.model === 'string' ? body.model : undefined,
+ hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,
+ operation:
+ body.operation === 'completion_cancel' || body.operation === 'connection_test'
+ ? body.operation
+ : 'completion_stream',
+ },
+ productionRedeemDeps(materializer)
+ )
res.status(200).json(result)
} catch (err) {
if (err instanceof LlmProviderAttemptRedeemError) {
@@ -100,16 +108,19 @@ export function createInternalLlmProviderAttemptRoutes(): Router {
asyncHandler(async (req, res) => {
const body = isPlainObject(req.body) ? req.body : {}
try {
- const result = await redeemGrokProviderAttempt({
- executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',
- requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',
- model: typeof body.model === 'string' ? body.model : undefined,
- hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,
- operation:
- body.operation === 'completion_cancel' || body.operation === 'connection_test'
- ? body.operation
- : 'completion_stream',
- })
+ const result = await redeemGrokProviderAttempt(
+ {
+ executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',
+ requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',
+ model: typeof body.model === 'string' ? body.model : undefined,
+ hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,
+ operation:
+ body.operation === 'completion_cancel' || body.operation === 'connection_test'
+ ? body.operation
+ : 'completion_stream',
+ },
+ productionGrokRedeemDeps(materializer)
+ )
res.status(200).json(result)
} catch (err) {
if (err instanceof GrokProviderAttemptRedeemError) {
diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.ts
index 7bc27b11e..c4ae09ce3 100644
--- a/control-api/src/services/grokProviderAttemptRedemption.ts
+++ b/control-api/src/services/grokProviderAttemptRedemption.ts
@@ -7,12 +7,17 @@ import { config } from '../config.js'
import { type DbClient, pool, withTransaction } from '../db.js'
import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'
import { rootLogger } from '../observability/logger.js'
+import { llmAllowlistConfigMapWriteFailuresTotal } from '../observability/metrics.js'
import { verifyGrokExecutionTicket } from './grokProviderAttemptTicket.js'
import {
getSafeGrokSubscriptionConnectionById,
loadGrokSubscriptionSecrets,
} from './grokSubscriptionConnection.js'
import { GrokSubscriptionOAuthError, ensureFreshGrokAccessToken } from './grokSubscriptionOAuth.js'
+import {
+ type AllowedModelsConfigMapMaterializer,
+ publishAllowedModelsConfigMapAfterGrantChange,
+} from './llmAllowedModelsConfigMap.js'
import { opaqueAttemptReceipt } from './llmProviderAttemptRedemption.js'
import {
loadLlmProviderAttempt,
@@ -78,9 +83,13 @@ export type RedeemGrokAttemptDeps = {
getConnectionById: typeof getSafeGrokSubscriptionConnectionById
encryptionKey: Buffer
ensureFreshAccessToken?: (connectionKey?: string) => Promise<void>
+ /** Republishes the runtime allowlist ConfigMap after a grant change. */
+ publishAllowlist: () => Promise<void>
}
-const defaultRedeemDeps = (): RedeemGrokAttemptDeps => ({
+export const productionGrokRedeemDeps = (
+ materializer: AllowedModelsConfigMapMaterializer
+): RedeemGrokAttemptDeps => ({
enabled: config.grokSubscriptionEnabled,
db: pool,
withTransaction,
@@ -96,11 +105,35 @@ const defaultRedeemDeps = (): RedeemGrokAttemptDeps => ({
enabled: config.grokSubscriptionEnabled,
connectionKey: connectionKey ?? '',
}),
+ publishAllowlist: async () => {
+ await publishAllowedModelsConfigMapAfterGrantChange(materializer)
+ },
})
+/**
+ * A refresh failure that persisted a connection status (a rejected refresh
+ * token marks the row reauth_required) changed what the runtime ConfigMap
+ * must say about the grant, so it is published before the redemption fails,
+ * as the refresh and catalog-sync routes do. A failed publish is counted and
+ * logged exactly as those routes count and log it; the redemption then
+ * fails with its own code, because the proxy contract has no code for an
+ * unwritten ConfigMap.
+ */
+async function publishAfterPersistedRefreshFailure(deps: RedeemGrokAttemptDeps): Promise<void> {
+ try {
+ await deps.publishAllowlist()
+ } catch (err) {
+ llmAllowlistConfigMapWriteFailuresTotal.inc({ phase: 'mutation' })
+ log.error(
+ { err, event: 'grok_allowed_models_cm_write_failed' },
+ 'Grok refresh failure persisted a connection status but the runtime ConfigMap was not updated'
+ )
+ }
+}
+
export async function redeemGrokProviderAttempt(
input: RedeemGrokAttemptInput,
- deps: RedeemGrokAttemptDeps = defaultRedeemDeps()
+ deps: RedeemGrokAttemptDeps
): Promise<RedeemGrokAttemptSuccess> {
if (!deps.enabled) {
throw new GrokProviderAttemptRedeemError('disabled', 'Grok subscription is disabled')
@@ -161,6 +194,9 @@ export async function redeemGrokProviderAttempt(
await deps.ensureFreshAccessToken(assigned.connectionKey)
} catch (err) {
if (err instanceof GrokSubscriptionOAuthError) {
+ if (err.persistedConnectionStatus === true) {
+ await publishAfterPersistedRefreshFailure(deps)
+ }
throw new GrokProviderAttemptRedeemError(
err.code === 'no_grant' || err.code === 'not_connected' || err.code === 'reauth_required'
? 'no_grant'
diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.ts
index c23b8fa21..5c8d451ad 100644
--- a/control-api/src/services/llmProviderAttemptRedemption.ts
+++ b/control-api/src/services/llmProviderAttemptRedemption.ts
@@ -3,6 +3,7 @@ import { config } from '../config.js'
import { type DbClient, pool, withTransaction } from '../db.js'
import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'
import { rootLogger } from '../observability/logger.js'
+import { llmAllowlistConfigMapWriteFailuresTotal } from '../observability/metrics.js'
import { chatgptAccountIdFromJwt } from './chatgptAccountId.js'
import {
getSafeCodexSubscriptionConnectionById,
@@ -12,6 +13,10 @@ import {
CodexSubscriptionOAuthError,
ensureFreshCodexAccessToken,
} from './codexSubscriptionOAuth.js'
+import {
+ type AllowedModelsConfigMapMaterializer,
+ publishAllowedModelsConfigMapAfterGrantChange,
+} from './llmAllowedModelsConfigMap.js'
import {
loadLlmProviderAttempt,
lockLlmProviderAttemptTicket,
@@ -88,9 +93,13 @@ export type RedeemAttemptDeps = {
getConnectionById: typeof getSafeCodexSubscriptionConnectionById
encryptionKey: Buffer
ensureFreshAccessToken?: (connectionKey?: string) => Promise<void>
+ /** Republishes the runtime allowlist ConfigMap after a grant change. */
+ publishAllowlist: () => Promise<void>
}
-const defaultRedeemDeps = (): RedeemAttemptDeps => ({
+export const productionRedeemDeps = (
+ materializer: AllowedModelsConfigMapMaterializer
+): RedeemAttemptDeps => ({
enabled: config.codexSubscriptionEnabled,
db: pool,
withTransaction,
@@ -107,11 +116,34 @@ const defaultRedeemDeps = (): RedeemAttemptDeps => ({
enabled: config.codexSubscriptionEnabled,
connectionKey,
}),
+ publishAllowlist: async () => {
+ await publishAllowedModelsConfigMapAfterGrantChange(materializer)
+ },
})
+/**
+ * A refresh failure that persisted a connection status (a rejected refresh
+ * token marks the row reauth_required) changed what the runtime ConfigMap
+ * must say about the grant, so it is published before the redemption fails,
+ * as the refresh and catalog-sync routes do. A failed publish is counted and
+ * logged exactly as the refresh route counts and logs it; the redemption
+ * then fails with its own code, which already answers 503 to the proxy.
+ */
+async function publishAfterPersistedRefreshFailure(deps: RedeemAttemptDeps): Promise<void> {
+ try {
+ await deps.publishAllowlist()
+ } catch (err) {
+ llmAllowlistConfigMapWriteFailuresTotal.inc({ phase: 'mutation' })
+ log.error(
+ { err, event: 'codex_allowed_models_cm_write_failed' },
+ 'Codex refresh failure persisted a connection status but the runtime ConfigMap was not updated'
+ )
+ }
+}
+
export async function redeemLlmProviderAttempt(
input: RedeemAttemptInput,
- deps: RedeemAttemptDeps = defaultRedeemDeps()
+ deps: RedeemAttemptDeps
): Promise<RedeemAttemptSuccess> {
if (!deps.enabled) {
throw new LlmProviderAttemptRedeemError('disabled', 'Codex subscription is disabled')
@@ -143,6 +175,9 @@ export async function redeemLlmProviderAttempt(
await deps.ensureFreshAccessToken(refreshKey)
} catch (err) {
if (err instanceof CodexSubscriptionOAuthError) {
+ if (err.persistedConnectionStatus === true) {
+ await publishAfterPersistedRefreshFailure(deps)
+ }
throw new LlmProviderAttemptRedeemError(
err.code === 'no_grant' || err.code === 'not_connected'
? 'no_grant'