Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: mark a rejected Codex refresh reauth_required (#753)

Alfredo Lopezd5729ca2ed99
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 211f524b5..aa041be5d 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 64d00e152..adeb0d846 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.ts
index 4ba0d6058..077d43950 100644
--- a/control-api/src/routes/admin/codexSubscription.ts
+++ b/control-api/src/routes/admin/codexSubscription.ts
@@ -754,6 +754,12 @@ export function createAdminCodexSubscriptionRouter(
})
return
}
+ // Same rule and same order as the Grok branch above: a rejected refresh
+ // token moves the row to `reauth_required` before the sync fails, so the
+ // publish comes before any error response.
+ if (syncOutcomeChangedTheRow(synced)) {
+ if (await publishRuntimeAllowlistOrFail(res)) return
+ }
if (synced.reason === 'no_grant' || synced.reason === 'disabled') {
res.status(404).json({ error: synced.reason })
return
@@ -766,14 +772,6 @@ export function createAdminCodexSubscriptionRouter(
sendOAuthError(res, new CodexSubscriptionOAuthError(synced.reason, synced.reason))
return
}
- // Same rule as the Grok branch above. No Codex error path writes the
- // connection row — its only status writes are `cancelled` and `revoked`,
- // both explicit operator actions — so in practice this publishes exactly
- // when `catalogStatus` recorded something, and skips the write when the
- // sync recorded nothing.
- if (syncOutcomeChangedTheRow(synced)) {
- if (await publishRuntimeAllowlistOrFail(res)) return
- }
res.status(503).json({
error: 'catalog_sync_failed',
// Report what the sync reported. Rewriting `never_synced` as
diff --git a/control-api/src/services/codexSubscriptionConnection.ts b/control-api/src/services/codexSubscriptionConnection.ts
index 81abdb6dd..7d25aa16e 100644
--- a/control-api/src/services/codexSubscriptionConnection.ts
+++ b/control-api/src/services/codexSubscriptionConnection.ts
@@ -647,6 +647,29 @@ export async function updateCodexAccessTokenInPlace(
return toSafeConnection(row)
}
+/**
+ * The vendor rejected the stored refresh token. Fenced on the revision the
+ * refresh read, so a grant replaced in the meantime is left alone.
+ */
+export async function markCodexRefreshRejected(
+ db: DbClient,
+ connectionKey: string,
+ expectedRevision: number
+): Promise<CodexSubscriptionSafeConnection | null> {
+ const result = await db.query(
+ `UPDATE codex_subscription_connections
+ SET status = 'reauth_required',
+ updated_at = now()
+ WHERE connection_key = $1
+ AND credential_revision = $2
+ AND revoked_at IS NULL
+ RETURNING ${SAFE_CONNECTION_COLUMNS}`,
+ [normalizeCodexConnectionKey(connectionKey), expectedRevision]
+ )
+ const row = result.rows[0] as SafeConnectionRow | undefined
+ return row ? toSafeConnection(row) : null
+}
+
export async function persistCodexChatgptAccountId(
db: DbClient,
encryptionKey: Buffer,
diff --git a/control-api/src/services/codexSubscriptionOAuth.ts b/control-api/src/services/codexSubscriptionOAuth.ts
index a3ff80f35..df68986fa 100644
--- a/control-api/src/services/codexSubscriptionOAuth.ts
+++ b/control-api/src/services/codexSubscriptionOAuth.ts
@@ -17,6 +17,7 @@ import {
insertInitialCodexSubscriptionConnection,
isCodexConnectionKeyConflict,
loadCodexSubscriptionSecrets,
+ markCodexRefreshRejected,
normalizeCodexConnectionKey,
persistCodexChatgptAccountId,
releaseCodexSubscriptionRefreshLock,
@@ -82,6 +83,7 @@ export type CodexOAuthErrorCode =
| 'browser_oauth_unregistered'
| 'fingerprint_in_use'
| 'connection_mismatch'
+ | 'reauth_required'
const CODEX_OAUTH_ERROR_CODES: ReadonlySet<CodexOAuthErrorCode> = new Set([
'disabled',
@@ -98,6 +100,7 @@ const CODEX_OAUTH_ERROR_CODES: ReadonlySet<CodexOAuthErrorCode> = new Set([
'browser_oauth_unregistered',
'fingerprint_in_use',
'connection_mismatch',
+ 'reauth_required',
])
export function isCodexOAuthErrorCode(value: string): value is CodexOAuthErrorCode {
@@ -106,11 +109,22 @@ export function isCodexOAuthErrorCode(value: string): value is CodexOAuthErrorCo
export class CodexSubscriptionOAuthError extends Error {
readonly code: CodexOAuthErrorCode
-
- constructor(code: CodexOAuthErrorCode, message: string) {
+ /**
+ * True when the connection row was written before this error was thrown: a
+ * refresh token the vendor rejected moves the row to `reauth_required` and
+ * only then does the error propagate, so the caller owes a republish.
+ */
+ readonly persistedConnectionStatus: boolean
+
+ constructor(
+ code: CodexOAuthErrorCode,
+ message: string,
+ opts: { persistedConnectionStatus?: boolean } = {}
+ ) {
super(message)
this.name = 'CodexSubscriptionOAuthError'
this.code = code
+ this.persistedConnectionStatus = opts.persistedConnectionStatus === true
}
}
@@ -188,6 +202,12 @@ export type CodexCatalogSyncResult =
refreshed?: number
staled?: number
connection?: CodexSubscriptionSafeConnection | null
+ /**
+ * The catalog was not synced but the CONNECTION row was written anyway.
+ * `catalogStatus` describes the catalog, so it cannot answer "did the row
+ * change?" — only this field can.
+ */
+ persisted?: boolean
}
function requireEnabled(deps: CodexOAuthDeps): void {
@@ -379,7 +399,10 @@ export async function refreshCodexSubscriptionConnection(
const secrets = await loadCodexSubscriptionSecrets(deps.db, deps.encryptionKey, key)
if (!secrets)
throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')
- const token = await exchangeRefreshToken(deps, secrets.refreshToken)
+ const token = await exchangeRefreshToken(deps, secrets.refreshToken, {
+ expectedRevision: secrets.credentialRevision,
+ connectionKey: key,
+ })
try {
return await rotateCodexSubscriptionCredentials(
deps.db,
@@ -628,7 +651,10 @@ export async function ensureFreshCodexAccessToken(deps: CodexOAuthDeps): Promise
if (!latest) {
throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')
}
- const token = await exchangeRefreshToken(deps, latest.refreshToken)
+ const token = await exchangeRefreshToken(deps, latest.refreshToken, {
+ expectedRevision: latest.credentialRevision,
+ connectionKey: key,
+ })
accountId = token.chatgptAccountId || chatgptAccountIdFromJwt(token.accessToken) || accountId
await updateCodexAccessTokenInPlace(
deps.db,
@@ -704,7 +730,9 @@ export async function runCodexCatalogSync(
)
const reason: CodexOAuthErrorCode | 'catalog_sync_failed' =
err instanceof CodexSubscriptionOAuthError ? err.code : 'catalog_sync_failed'
- return { ok: false, catalogStatus: 'never_synced', reason }
+ const persisted =
+ err instanceof CodexSubscriptionOAuthError && err.persistedConnectionStatus === true
+ return { ok: false, catalogStatus: 'never_synced', reason, persisted }
}
}
@@ -730,9 +758,14 @@ async function exchangeAuthorizationCode(
return parseTokenResponse(result.body)
}
+/**
+ * Both callers hold the refresh lock and pass the revision they read under it,
+ * so a rejection can be told apart from a refresh that lost a race.
+ */
async function exchangeRefreshToken(
deps: CodexOAuthDeps,
- refreshToken: string
+ refreshToken: string,
+ fence: { expectedRevision: number; connectionKey: string }
): Promise<ParsedCodexToken> {
const result = await postForm(deps, CODEX_OAUTH_TOKEN_URL, {
grant_type: 'refresh_token',
@@ -740,6 +773,30 @@ async function exchangeRefreshToken(
client_id: deps.clientId,
})
if (!result.ok) {
+ const error = readUpstreamErrorCode(result.body)
+ if ((result.status === 400 || result.status === 401) && error === 'invalid_grant') {
+ const latest = await loadCodexSubscriptionSecrets(
+ deps.db,
+ deps.encryptionKey,
+ fence.connectionKey
+ )
+ const current = await getSafeCodexSubscriptionConnection(deps.db, fence.connectionKey)
+ const lockChanged = !current?.refreshLockHeld
+ const revisionChanged = latest?.credentialRevision !== fence.expectedRevision
+ if (lockChanged || revisionChanged) {
+ throw new CodexSubscriptionOAuthError(
+ 'stale_revision',
+ 'invalid_grant observed after a lost refresh race'
+ )
+ }
+ await markCodexRefreshRejected(deps.db, fence.connectionKey, fence.expectedRevision)
+ // The row is now `reauth_required`. The throw below must not read as
+ // "nothing happened": the ConfigMap carries this status, so whoever
+ // catches this owes a republish.
+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
+ persistedConnectionStatus: true,
+ })
+ }
throw new CodexSubscriptionOAuthError('provider_unavailable', 'refresh token exchange failed')
}
return parseTokenResponse(result.body)
diff --git a/control-api/test/services.codexSubscriptionOAuth.test.ts b/control-api/test/services.codexSubscriptionOAuth.test.ts
index 71b5be7e9..8310de7ed 100644
--- a/control-api/test/services.codexSubscriptionOAuth.test.ts
+++ b/control-api/test/services.codexSubscriptionOAuth.test.ts
@@ -919,6 +919,7 @@ describe('codex subscription OAuth broker', () => {
ok: false,
catalogStatus: 'never_synced',
reason: 'catalog_sync_failed',
+ persisted: false,
})
})
@@ -940,6 +941,7 @@ describe('codex subscription OAuth broker', () => {
ok: false,
catalogStatus: 'never_synced',
reason: 'provider_unavailable',
+ persisted: false,
})
})
})
diff --git a/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts b/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts
index 248dc115e..e91b6f409 100644
--- a/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts
+++ b/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts
@@ -237,6 +237,7 @@ describeRealPostgres('Codex refresh rejected by the vendor on real PostgreSQL (#
materializer,
})
const callsAfterFirstTick = fetchFn.mock.calls.length
+ const publishesAfterFirstTick = published.length
const secondKeys: string[] = []
const second = await reconcileSubscriptionCatalogs({
@@ -254,8 +255,11 @@ describeRealPostgres('Codex refresh rejected by the vendor on real PostgreSQL (#
failed: 0,
skipped: live.length - connectedBefore.length,
})
- expect(published).toHaveLength(1)
+ expect(publishesAfterFirstTick).toBe(1)
expect(await statusOf(key)).toBe('reauth_required')
+ // The cron publishes on every tick by design (subscriptionCatalogSyncCron.ts),
+ // so tick 2 adds exactly one more.
+ expect(published).toHaveLength(2)
// Tick 2 listed every grant (all of them count as skipped) and sent no dead
// refresh token again.
expect(second).toMatchObject({ synced: 0, degraded: 0, failed: 0, skipped: live.length })