Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: mark a rejected Codex refresh reauth_required (#753)
Alfredo Lopezd5729ca2ed99
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 211f524b5..aa041be5d 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 64d00e152..adeb0d846 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.tsindex 4ba0d6058..077d43950 100644--- a/control-api/src/routes/admin/codexSubscription.ts+++ b/control-api/src/routes/admin/codexSubscription.ts@@ -754,6 +754,12 @@ export function createAdminCodexSubscriptionRouter(})return}+ // Same rule and same order as the Grok branch above: a rejected refresh+ // token moves the row to `reauth_required` before the sync fails, so the+ // publish comes before any error response.+ if (syncOutcomeChangedTheRow(synced)) {+ if (await publishRuntimeAllowlistOrFail(res)) return+ }if (synced.reason === 'no_grant' || synced.reason === 'disabled') {res.status(404).json({ error: synced.reason })return@@ -766,14 +772,6 @@ export function createAdminCodexSubscriptionRouter(sendOAuthError(res, new CodexSubscriptionOAuthError(synced.reason, synced.reason))return}- // Same rule as the Grok branch above. No Codex error path writes the- // connection row — its only status writes are `cancelled` and `revoked`,- // both explicit operator actions — so in practice this publishes exactly- // when `catalogStatus` recorded something, and skips the write when the- // sync recorded nothing.- if (syncOutcomeChangedTheRow(synced)) {- if (await publishRuntimeAllowlistOrFail(res)) return- }res.status(503).json({error: 'catalog_sync_failed',// Report what the sync reported. Rewriting `never_synced` asdiff --git a/control-api/src/services/codexSubscriptionConnection.ts b/control-api/src/services/codexSubscriptionConnection.tsindex 81abdb6dd..7d25aa16e 100644--- a/control-api/src/services/codexSubscriptionConnection.ts+++ b/control-api/src/services/codexSubscriptionConnection.ts@@ -647,6 +647,29 @@ export async function updateCodexAccessTokenInPlace(return toSafeConnection(row)}+/**+ * The vendor rejected the stored refresh token. Fenced on the revision the+ * refresh read, so a grant replaced in the meantime is left alone.+ */+export async function markCodexRefreshRejected(+ db: DbClient,+ connectionKey: string,+ expectedRevision: number+): Promise<CodexSubscriptionSafeConnection | null> {+ const result = await db.query(+ `UPDATE codex_subscription_connections+ SET status = 'reauth_required',+ updated_at = now()+ WHERE connection_key = $1+ AND credential_revision = $2+ AND revoked_at IS NULL+ RETURNING ${SAFE_CONNECTION_COLUMNS}`,+ [normalizeCodexConnectionKey(connectionKey), expectedRevision]+ )+ const row = result.rows[0] as SafeConnectionRow | undefined+ return row ? toSafeConnection(row) : null+}+export async function persistCodexChatgptAccountId(db: DbClient,encryptionKey: Buffer,diff --git a/control-api/src/services/codexSubscriptionOAuth.ts b/control-api/src/services/codexSubscriptionOAuth.tsindex a3ff80f35..df68986fa 100644--- a/control-api/src/services/codexSubscriptionOAuth.ts+++ b/control-api/src/services/codexSubscriptionOAuth.ts@@ -17,6 +17,7 @@ import {insertInitialCodexSubscriptionConnection,isCodexConnectionKeyConflict,loadCodexSubscriptionSecrets,+ markCodexRefreshRejected,normalizeCodexConnectionKey,persistCodexChatgptAccountId,releaseCodexSubscriptionRefreshLock,@@ -82,6 +83,7 @@ export type CodexOAuthErrorCode =| 'browser_oauth_unregistered'| 'fingerprint_in_use'| 'connection_mismatch'+ | 'reauth_required'const CODEX_OAUTH_ERROR_CODES: ReadonlySet<CodexOAuthErrorCode> = new Set(['disabled',@@ -98,6 +100,7 @@ const CODEX_OAUTH_ERROR_CODES: ReadonlySet<CodexOAuthErrorCode> = new Set(['browser_oauth_unregistered','fingerprint_in_use','connection_mismatch',+ 'reauth_required',])export function isCodexOAuthErrorCode(value: string): value is CodexOAuthErrorCode {@@ -106,11 +109,22 @@ export function isCodexOAuthErrorCode(value: string): value is CodexOAuthErrorCoexport class CodexSubscriptionOAuthError extends Error {readonly code: CodexOAuthErrorCode-- constructor(code: CodexOAuthErrorCode, message: string) {+ /**+ * True when the connection row was written before this error was thrown: a+ * refresh token the vendor rejected moves the row to `reauth_required` and+ * only then does the error propagate, so the caller owes a republish.+ */+ readonly persistedConnectionStatus: boolean++ constructor(+ code: CodexOAuthErrorCode,+ message: string,+ opts: { persistedConnectionStatus?: boolean } = {}+ ) {super(message)this.name = 'CodexSubscriptionOAuthError'this.code = code+ this.persistedConnectionStatus = opts.persistedConnectionStatus === true}}@@ -188,6 +202,12 @@ export type CodexCatalogSyncResult =refreshed?: numberstaled?: numberconnection?: CodexSubscriptionSafeConnection | null+ /**+ * The catalog was not synced but the CONNECTION row was written anyway.+ * `catalogStatus` describes the catalog, so it cannot answer "did the row+ * change?" — only this field can.+ */+ persisted?: boolean}function requireEnabled(deps: CodexOAuthDeps): void {@@ -379,7 +399,10 @@ export async function refreshCodexSubscriptionConnection(const secrets = await loadCodexSubscriptionSecrets(deps.db, deps.encryptionKey, key)if (!secrets)throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')- const token = await exchangeRefreshToken(deps, secrets.refreshToken)+ const token = await exchangeRefreshToken(deps, secrets.refreshToken, {+ expectedRevision: secrets.credentialRevision,+ connectionKey: key,+ })try {return await rotateCodexSubscriptionCredentials(deps.db,@@ -628,7 +651,10 @@ export async function ensureFreshCodexAccessToken(deps: CodexOAuthDeps): Promiseif (!latest) {throw new CodexSubscriptionOAuthError('no_grant', 'encrypted refresh token missing')}- const token = await exchangeRefreshToken(deps, latest.refreshToken)+ const token = await exchangeRefreshToken(deps, latest.refreshToken, {+ expectedRevision: latest.credentialRevision,+ connectionKey: key,+ })accountId = token.chatgptAccountId || chatgptAccountIdFromJwt(token.accessToken) || accountIdawait updateCodexAccessTokenInPlace(deps.db,@@ -704,7 +730,9 @@ export async function runCodexCatalogSync()const reason: CodexOAuthErrorCode | 'catalog_sync_failed' =err instanceof CodexSubscriptionOAuthError ? err.code : 'catalog_sync_failed'- return { ok: false, catalogStatus: 'never_synced', reason }+ const persisted =+ err instanceof CodexSubscriptionOAuthError && err.persistedConnectionStatus === true+ return { ok: false, catalogStatus: 'never_synced', reason, persisted }}}@@ -730,9 +758,14 @@ async function exchangeAuthorizationCode(return parseTokenResponse(result.body)}+/**+ * Both callers hold the refresh lock and pass the revision they read under it,+ * so a rejection can be told apart from a refresh that lost a race.+ */async function exchangeRefreshToken(deps: CodexOAuthDeps,- refreshToken: string+ refreshToken: string,+ fence: { expectedRevision: number; connectionKey: string }): Promise<ParsedCodexToken> {const result = await postForm(deps, CODEX_OAUTH_TOKEN_URL, {grant_type: 'refresh_token',@@ -740,6 +773,30 @@ async function exchangeRefreshToken(client_id: deps.clientId,})if (!result.ok) {+ const error = readUpstreamErrorCode(result.body)+ if ((result.status === 400 || result.status === 401) && error === 'invalid_grant') {+ const latest = await loadCodexSubscriptionSecrets(+ deps.db,+ deps.encryptionKey,+ fence.connectionKey+ )+ const current = await getSafeCodexSubscriptionConnection(deps.db, fence.connectionKey)+ const lockChanged = !current?.refreshLockHeld+ const revisionChanged = latest?.credentialRevision !== fence.expectedRevision+ if (lockChanged || revisionChanged) {+ throw new CodexSubscriptionOAuthError(+ 'stale_revision',+ 'invalid_grant observed after a lost refresh race'+ )+ }+ await markCodexRefreshRejected(deps.db, fence.connectionKey, fence.expectedRevision)+ // The row is now `reauth_required`. The throw below must not read as+ // "nothing happened": the ConfigMap carries this status, so whoever+ // catches this owes a republish.+ throw new CodexSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {+ persistedConnectionStatus: true,+ })+ }throw new CodexSubscriptionOAuthError('provider_unavailable', 'refresh token exchange failed')}return parseTokenResponse(result.body)diff --git a/control-api/test/services.codexSubscriptionOAuth.test.ts b/control-api/test/services.codexSubscriptionOAuth.test.tsindex 71b5be7e9..8310de7ed 100644--- a/control-api/test/services.codexSubscriptionOAuth.test.ts+++ b/control-api/test/services.codexSubscriptionOAuth.test.ts@@ -919,6 +919,7 @@ describe('codex subscription OAuth broker', () => {ok: false,catalogStatus: 'never_synced',reason: 'catalog_sync_failed',+ persisted: false,})})@@ -940,6 +941,7 @@ describe('codex subscription OAuth broker', () => {ok: false,catalogStatus: 'never_synced',reason: 'provider_unavailable',+ persisted: false,})})})diff --git a/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts b/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.tsindex 248dc115e..e91b6f409 100644--- a/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts+++ b/control-api/test/services.codexSubscriptionRefreshRejected.realPostgres.integration.test.ts@@ -237,6 +237,7 @@ describeRealPostgres('Codex refresh rejected by the vendor on real PostgreSQL (#materializer,})const callsAfterFirstTick = fetchFn.mock.calls.length+ const publishesAfterFirstTick = published.lengthconst secondKeys: string[] = []const second = await reconcileSubscriptionCatalogs({@@ -254,8 +255,11 @@ describeRealPostgres('Codex refresh rejected by the vendor on real PostgreSQL (#failed: 0,skipped: live.length - connectedBefore.length,})- expect(published).toHaveLength(1)+ expect(publishesAfterFirstTick).toBe(1)expect(await statusOf(key)).toBe('reauth_required')+ // The cron publishes on every tick by design (subscriptionCatalogSyncCron.ts),+ // so tick 2 adds exactly one more.+ expect(published).toHaveLength(2)// Tick 2 listed every grant (all of them count as skipped) and sent no dead// refresh token again.expect(second).toMatchObject({ synced: 0, degraded: 0, failed: 0, skipped: live.length })