Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(codex): close CI typecheck and remaining grant/refresh races

Alfredo Lopez368aa8a7fa08
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 8c09ceb3b..1c219f88f 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.518",
+ "version": "0.1.519",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.518",
+ "version": "0.1.519",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 53787d300..560a05f10 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.518",
+ "version": "0.1.519",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/admin/pluginWorkloadSdk.ts b/control-api/src/routes/admin/pluginWorkloadSdk.ts
index d1eb95ecf..de166e0b4 100644
--- a/control-api/src/routes/admin/pluginWorkloadSdk.ts
+++ b/control-api/src/routes/admin/pluginWorkloadSdk.ts
@@ -375,6 +375,51 @@ export interface AdminPluginWorkloadSdkRouterDeps {
const log = rootLogger.child({ module: 'admin-plugin-workload-sdk' })
+async function publishPromptBridgeGrantIdentity(input: {
+ gateway: Pick<K8sGateway, 'getResource' | 'updateResource'>
+ recipeNamespace: string
+ recipeName: string
+ nextRef: string
+}): Promise<{ error?: { status: number; error: string } }> {
+ let next = input.nextRef
+ if (!next) {
+ let recipeAgent: string | undefined
+ try {
+ const recipe = (await input.gateway.getResource(
+ 'workflowrecipes',
+ input.recipeName,
+ input.recipeNamespace
+ )) as { spec?: { agent?: { provider?: string } } }
+ recipeAgent =
+ typeof recipe.spec?.agent?.provider === 'string' ? recipe.spec.agent.provider : undefined
+ } catch (err) {
+ log.error(
+ { err, recipeNamespace: input.recipeNamespace, recipeName: input.recipeName },
+ 'failed to read WorkflowRecipe before publishing Codex grant identity'
+ )
+ return { error: { status: 503, error: 'recipe_annotation_publish_failed' } }
+ }
+ if (recipeAgent === 'codex-subscription') {
+ return {}
+ }
+ next = 'unassigned'
+ }
+ try {
+ await publishRecipeGrantIdentity({
+ gateway: input.gateway,
+ namespace: input.recipeNamespace,
+ name: input.recipeName,
+ next,
+ })
+ } catch (err) {
+ if (err instanceof RecipeCodexGrantIdentityError) {
+ return { error: { status: err.status, error: err.error } }
+ }
+ throw err
+ }
+ return {}
+}
+
export function createAdminPluginWorkloadSdkRouter(
deps: AdminPluginWorkloadSdkRouterDeps = {}
): Router {
@@ -555,20 +600,17 @@ export function createAdminPluginWorkloadSdkRouter(
target.connectionRef &&
!isCodexUnassignedConnectionKey(target.connectionRef)
)?.connectionRef ?? ''
- if (capabilityFamily === 'promptBridge' && deps.gateway && brokerConnectionRef) {
- try {
- await publishRecipeGrantIdentity({
- gateway: deps.gateway,
- namespace: recipeNamespace,
- name: recipeName,
- next: brokerConnectionRef,
- })
- } catch (err) {
- if (err instanceof RecipeCodexGrantIdentityError) {
- res.status(err.status).json({ error: err.error })
- return
- }
- throw err
+ const hasBrokerTarget = promptTargets.some(target => isBrokerBackedProvider(target.provider))
+ if (capabilityFamily === 'promptBridge' && deps.gateway && hasBrokerTarget) {
+ const published = await publishPromptBridgeGrantIdentity({
+ gateway: deps.gateway,
+ recipeNamespace,
+ recipeName,
+ nextRef: brokerConnectionRef,
+ })
+ if (published.error) {
+ res.status(published.error.status).json({ error: published.error.error })
+ return
}
}
@@ -829,42 +871,16 @@ export function createAdminPluginWorkloadSdkRouter(
target.connectionRef &&
!isCodexUnassignedConnectionKey(target.connectionRef)
)?.connectionRef ?? ''
- let recipeAgent: string | undefined
- try {
- const recipe = (await deps.gateway.getResource(
- 'workflowrecipes',
- recipeName,
- recipeNamespace
- )) as { spec?: { agent?: { provider?: string } } }
- recipeAgent =
- typeof recipe.spec?.agent?.provider === 'string'
- ? recipe.spec.agent.provider
- : undefined
- } catch (err) {
- log.error(
- { err, recipeNamespace, recipeName },
- 'failed to read WorkflowRecipe after grant delete'
- )
- res.status(503).json({ error: 'recipe_annotation_publish_failed' })
+ const published = await publishPromptBridgeGrantIdentity({
+ gateway: deps.gateway,
+ recipeNamespace,
+ recipeName,
+ nextRef: remainingRef,
+ })
+ if (published.error) {
+ res.status(published.error.status).json({ error: published.error.error })
return
}
- const shouldUnassign = !remainingRef && recipeAgent !== 'codex-subscription'
- if (remainingRef || shouldUnassign) {
- try {
- await publishRecipeGrantIdentity({
- gateway: deps.gateway,
- namespace: recipeNamespace,
- name: recipeName,
- next: remainingRef || 'unassigned',
- })
- } catch (err) {
- if (err instanceof RecipeCodexGrantIdentityError) {
- res.status(err.status).json({ error: err.error })
- return
- }
- throw err
- }
- }
}
res.status(200).json({ deleted: true })
})
diff --git a/control-api/src/services/codexSubscriptionOAuth.ts b/control-api/src/services/codexSubscriptionOAuth.ts
index aeddd285b..fcf2587b5 100644
--- a/control-api/src/services/codexSubscriptionOAuth.ts
+++ b/control-api/src/services/codexSubscriptionOAuth.ts
@@ -44,9 +44,22 @@ const DEVICE_CODE_TIMEOUT_SECONDS = 15 * 60
const TOKEN_TIMEOUT_MS = 15_000
const REFRESH_LOCK_TTL_MS = 30_000
+const REFRESH_LOCK_POLL_MS = 200
const BROWSER_STATE_TTL_MS = 10 * 60_000
const ACCESS_TOKEN_REFRESH_SKEW_MS = 5 * 60_000
+function accessTokenIsUsable(secrets: {
+ accessToken?: string | null
+ accessTokenExpiresAt?: Date | null
+}): boolean {
+ if (!secrets.accessToken) return false
+ return secrets.accessTokenExpiresAt == null || secrets.accessTokenExpiresAt.getTime() > Date.now()
+}
+
+function sleep(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms))
+}
+
export type CodexOAuthErrorCode =
| 'disabled'
| 'not_connected'
@@ -477,29 +490,42 @@ export async function ensureFreshCodexAccessToken(deps: CodexOAuthDeps): Promise
return
}
- const lockToken = randomBytes(16).toString('hex')
- const locked = await acquireCodexSubscriptionRefreshLock(
+ let lockToken = randomBytes(16).toString('hex')
+ let locked = await acquireCodexSubscriptionRefreshLock(
deps.db,
lockToken,
REFRESH_LOCK_TTL_MS,
key
)
if (!locked) {
- await new Promise(resolve => setTimeout(resolve, 400))
- const latest = await loadCodexSubscriptionSecrets(deps.db, deps.encryptionKey, key)
- if (!latest) {
- throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')
- }
- const waitedAccountId = latest.chatgptAccountId || chatgptAccountIdFromJwt(latest.accessToken)
- const tokenStillValid =
- latest.accessTokenExpiresAt == null || latest.accessTokenExpiresAt.getTime() > Date.now()
- if (latest.accessToken && tokenStillValid) {
- if (waitedAccountId && !latest.chatgptAccountId) {
- await persistCodexChatgptAccountId(deps.db, deps.encryptionKey, waitedAccountId, key)
+ // The winner's upstream token exchange takes seconds. A single 400 ms
+ // peek loses every concurrent completion at the hourly expiry boundary.
+ const deadline = Date.now() + REFRESH_LOCK_TTL_MS
+ while (true) {
+ await sleep(REFRESH_LOCK_POLL_MS)
+ const latest = await loadCodexSubscriptionSecrets(deps.db, deps.encryptionKey, key)
+ if (!latest) {
+ throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')
+ }
+ const waitedAccountId = latest.chatgptAccountId || chatgptAccountIdFromJwt(latest.accessToken)
+ if (accessTokenIsUsable(latest)) {
+ if (waitedAccountId && !latest.chatgptAccountId) {
+ await persistCodexChatgptAccountId(deps.db, deps.encryptionKey, waitedAccountId, key)
+ }
+ return
+ }
+ if (Date.now() >= deadline) {
+ throw new CodexSubscriptionOAuthError('refresh_in_flight', 'another refresh holds the lock')
}
- return
+ lockToken = randomBytes(16).toString('hex')
+ locked = await acquireCodexSubscriptionRefreshLock(
+ deps.db,
+ lockToken,
+ REFRESH_LOCK_TTL_MS,
+ key
+ )
+ if (locked) break
}
- throw new CodexSubscriptionOAuthError('refresh_in_flight', 'another refresh holds the lock')
}
try {
const latest = await loadCodexSubscriptionSecrets(deps.db, deps.encryptionKey, key)
diff --git a/control-api/src/services/recipeCodexGrantIdentity.ts b/control-api/src/services/recipeCodexGrantIdentity.ts
index 3a3bbc8fa..461262785 100644
--- a/control-api/src/services/recipeCodexGrantIdentity.ts
+++ b/control-api/src/services/recipeCodexGrantIdentity.ts
@@ -89,15 +89,14 @@ export async function publishRecipeGrantIdentity(input: {
[CODEX_CONNECTION_REF_ANNOTATION]: next === CODEX_UNASSIGNED_CONNECTION_KEY ? '' : next,
}
const spec = asRecord(current.spec) ?? {}
+ const labels = stringMap(current.metadata?.labels)
try {
await input.gateway.updateResource(
'workflowrecipes',
input.name,
{
metadata: {
- ...(stringMap(current.metadata?.labels)
- ? { labels: stringMap(current.metadata?.labels) }
- : {}),
+ ...(Object.keys(labels).length > 0 ? { labels } : {}),
annotations: nextAnnotations,
...(current.metadata?.resourceVersion
? { resourceVersion: current.metadata.resourceVersion }
diff --git a/control-api/test/routes.adminPluginWorkloadSdk.test.ts b/control-api/test/routes.adminPluginWorkloadSdk.test.ts
index 28c370188..f7188af8a 100644
--- a/control-api/test/routes.adminPluginWorkloadSdk.test.ts
+++ b/control-api/test/routes.adminPluginWorkloadSdk.test.ts
@@ -726,7 +726,12 @@ describe('routes/admin/pluginWorkloadSdk — grants', () => {
.send({ ...codexGrantBody, promptTargets: [targetWithoutConnection] })
expect(res.status).toBe(200)
expect(isCodexAssignmentAllowed).not.toHaveBeenCalled()
- expect(publishRecipeGrantIdentity).not.toHaveBeenCalled()
+ expect(publishRecipeGrantIdentity).toHaveBeenCalledWith({
+ gateway: { getResource, updateResource },
+ namespace: 'sandbox-recipes',
+ name: 'sdk-recipe',
+ next: 'unassigned',
+ })
expect(sdkDb.upsertGrant).toHaveBeenCalled()
})
@@ -740,6 +745,27 @@ describe('routes/admin/pluginWorkloadSdk — grants', () => {
})
expect(res.status).toBe(200)
expect(isCodexAssignmentAllowed).not.toHaveBeenCalled()
+ expect(publishRecipeGrantIdentity).toHaveBeenCalledWith({
+ gateway: { getResource, updateResource },
+ namespace: 'sandbox-recipes',
+ name: 'sdk-recipe',
+ next: 'unassigned',
+ })
+ expect(sdkDb.upsertGrant).toHaveBeenCalled()
+ })
+
+ it('does not unassign a Codex-agent recipe when upsert stores an unassigned target', async () => {
+ vi.mocked(sdkDb.upsertGrant).mockResolvedValue({ id: 'g-legacy' } as never)
+ getResource.mockResolvedValue({
+ spec: { agent: { provider: 'codex-subscription', model: 'gpt-5.3-codex' } },
+ })
+ const res = await request(buildApp())
+ .post('/admin/plugin-workload-sdk/grants')
+ .send({
+ ...codexGrantBody,
+ promptTargets: [{ ...codexGrantBody.promptTargets[0]!, connectionRef: 'unassigned' }],
+ })
+ expect(res.status).toBe(200)
expect(publishRecipeGrantIdentity).not.toHaveBeenCalled()
expect(sdkDb.upsertGrant).toHaveBeenCalled()
})
diff --git a/control-api/test/services.codexSubscriptionOAuth.test.ts b/control-api/test/services.codexSubscriptionOAuth.test.ts
index 5e4672f84..f334c0a57 100644
--- a/control-api/test/services.codexSubscriptionOAuth.test.ts
+++ b/control-api/test/services.codexSubscriptionOAuth.test.ts
@@ -516,18 +516,48 @@ describe('codex subscription OAuth broker', () => {
expect(repos.updateInPlace).not.toHaveBeenCalled()
})
- it('fails loud when the refresh lock is held and the stored token is already expired', async () => {
- repos.loadSecrets.mockResolvedValue({
+ it('fails loud when the refresh lock is held and the stored token stays expired', async () => {
+ vi.useFakeTimers()
+ try {
+ repos.loadSecrets.mockResolvedValue({
+ refreshToken: 'refresh-secret',
+ accessToken: 'stale-access',
+ accessTokenExpiresAt: new Date(Date.now() - 1_000),
+ chatgptAccountId: null,
+ credentialRevision: 3,
+ })
+ repos.acquireLock.mockResolvedValue(false)
+ const pending = expect(ensureFreshCodexAccessToken(deps(vi.fn()))).rejects.toMatchObject({
+ code: 'refresh_in_flight',
+ })
+ await vi.advanceTimersByTimeAsync(30_000)
+ await pending
+ expect(repos.updateInPlace).not.toHaveBeenCalled()
+ } finally {
+ await vi.runOnlyPendingTimersAsync()
+ vi.useRealTimers()
+ }
+ })
+
+ it('waits for a concurrent refresh to publish a usable token instead of failing the losers', async () => {
+ const expired = {
refreshToken: 'refresh-secret',
accessToken: 'stale-access',
accessTokenExpiresAt: new Date(Date.now() - 1_000),
chatgptAccountId: null,
credentialRevision: 3,
- })
+ }
repos.acquireLock.mockResolvedValue(false)
- await expect(ensureFreshCodexAccessToken(deps(vi.fn()))).rejects.toMatchObject({
- code: 'refresh_in_flight',
- })
+ repos.loadSecrets
+ .mockResolvedValueOnce(expired)
+ .mockResolvedValueOnce(expired)
+ .mockResolvedValueOnce({
+ ...expired,
+ accessToken: 'access-secret',
+ accessTokenExpiresAt: new Date(Date.now() + 3_600_000),
+ chatgptAccountId: 'acct_live_9',
+ })
+ await expect(ensureFreshCodexAccessToken(deps(vi.fn()))).resolves.toBeUndefined()
expect(repos.updateInPlace).not.toHaveBeenCalled()
})
diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
index 95467547a..2925f16d9 100644
--- a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
@@ -113,6 +113,18 @@ describe('authorizeLlmProviderAttempt', () => {
executionTicket: 'ticket.jwt',
requestHash: hashCodexCompletionRequestV1(REQUEST),
})
+ expect(current.evaluateBudget).toHaveBeenCalledWith(
+ expect.objectContaining({
+ task_ref: 'invocation-1:1:1',
+ context_ref: null,
+ team_id: null,
+ user_id: null,
+ source_kind: 'channel',
+ }),
+ expect.anything(),
+ expect.anything(),
+ expect.objectContaining({ requiredUnit: 'tokens' })
+ )
expect(current.insertAttempt).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index ae1ebcaa8..44aac9ce3 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.263",
+ "version": "0.1.264",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.263",
+ "version": "0.1.264",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 6fe84338f..0b094d4ce 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.263",
+ "version": "0.1.264",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/hostRoleNoopGate.test.ts b/host-context-controller/src/__tests__/hostRoleNoopGate.test.ts
index a2b1cc7bd..4eb9082b6 100644
--- a/host-context-controller/src/__tests__/hostRoleNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/hostRoleNoopGate.test.ts
@@ -102,7 +102,11 @@ describe('Host ensureHostRole no-op gate', () => {
let existing: k8s.V1Role | undefined
rbacApi.readNamespacedRole.mockImplementation(() => {
const live = structuredClone(desiredFromCreate(rbacApi))
- secretGetRule(live).resourceNames![0] = host.spec.secretRef
+ const currentSecretRef = host.spec.secretRef
+ if (!currentSecretRef) {
+ throw new Error('test host is missing secretRef')
+ }
+ secretGetRule(live).resourceNames![0] = currentSecretRef
existing = asApiserverRole(live)
return Promise.resolve(existing)
})
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index dd0830250..8f868668f 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.318",
+ "version": "0.1.319",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.318",
+ "version": "0.1.319",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 9dad4af3a..bf6ead106 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.318",
+ "version": "0.1.319",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 8825f9331..9b7569c21 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -977,9 +977,9 @@ export class WorkflowReconciler {
/**
* Per-recipe snapshot: re-parse the last allowlist ConfigMap with the
- * recipe's assigned grant key. A multi-grant map fail-closes for a missing
- * key (including the `unassigned` sentinel); a legacy flat ConfigMap keeps
- * the flat catalog. Read failures keep the fail-closed error snapshot.
+ * recipe's assigned grant key. `unassigned` never inherits a mapped or
+ * legacy catalog. A mapped miss fail-closes to an empty Codex snapshot.
+ * Read failures keep the fail-closed error snapshot.
*/
private codexSnapshotFor(connectionKey: string): CodexCatalogSnapshot {
return snapshotForAssignedCodexGrant(connectionKey, this.lastCodexConfigMap, this.codexSnapshot)