Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): close Codex grant lifecycle races and enforce attempt connection integrity

Jose Gabriel Ramirez Encinasea16f3762242
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index a521e06e8..c5cf42b54 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.575",
+ "version": "0.1.576",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.575",
+ "version": "0.1.576",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 96ca86b36..c14aecb9c 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.575",
+ "version": "0.1.576",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 3fdfaf294..5ca72d8ab 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -28,6 +28,7 @@ import {
} from './services/grokSubscriptionSchema.js'
import {
applyLlmProviderAttemptConnectionIdSchema,
+ applyLlmProviderAttemptConnectionIntegritySchema,
applyLlmProviderAttemptSchema,
applyLlmProviderAttemptSdkLinkOnDeleteSetNullSchema,
applyLlmProviderAttemptSdkLinkSchema,
@@ -6041,6 +6042,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
version: '0113_grok_subscription_terminal_connection_key',
apply: applyGrokSubscriptionTerminalConnectionKeySchema,
},
+ {
+ version: '0114_llm_provider_attempts_connection_integrity',
+ apply: applyLlmProviderAttemptConnectionIntegritySchema,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
diff --git a/control-api/src/services/codexSubscriptionCatalog.ts b/control-api/src/services/codexSubscriptionCatalog.ts
index 9e0dcb679..8d3c4d565 100644
--- a/control-api/src/services/codexSubscriptionCatalog.ts
+++ b/control-api/src/services/codexSubscriptionCatalog.ts
@@ -1,6 +1,6 @@
import jwt from 'jsonwebtoken'
import { config } from '../config.js'
-import type { DbClient } from '../db.js'
+import { type DbClient, withTransaction } from '../db.js'
import { rootLogger } from '../observability/logger.js'
import {
CODEX_SUBSCRIPTION_CONNECTION_KEY,
@@ -13,12 +13,58 @@ import {
readHostCodexConnectionRef,
recordCodexCatalogOutcome,
} from './codexSubscriptionConnection.js'
+import { boundDiscoveredCatalogModels } from './subscriptionCatalogBounds.js'
const log = rootLogger.child({ module: 'codex-subscription-catalog' })
const PROVIDER = 'codex-subscription'
export type CodexCatalogOutcome = 'ready' | 'auth-rejected' | 'unavailable'
+/**
+ * Runs `work` inside one database transaction and commits only when it
+ * resolves.
+ */
+export type CodexTransactionRunner = <T>(work: (tx: DbClient) => Promise<T>) => Promise<T>
+
+type PoolLike = DbClient & {
+ connect: () => Promise<DbClient & { release: (err?: Error | boolean) => void }>
+ totalCount: number
+}
+
+function isPoolLike(db: DbClient): db is PoolLike {
+ const candidate = db as Partial<PoolLike>
+ return typeof candidate.connect === 'function' && typeof candidate.totalCount === 'number'
+}
+
+/**
+ * Transaction runner for the DbClient a caller passed. A pg Pool gets a
+ * dedicated session from that same pool (real-PostgreSQL callers); any other
+ * DbClient (the admin routes' `{ query }` wrapper around the Control API pool)
+ * uses the Control API pool's `withTransaction`.
+ */
+export function codexTransactionRunnerFor(db: DbClient): CodexTransactionRunner {
+ if (!isPoolLike(db)) return work => withTransaction(work)
+ return async work => {
+ const client = await db.connect()
+ let releaseError: Error | boolean | undefined
+ try {
+ await client.query('BEGIN')
+ const result = await work(client)
+ await client.query('COMMIT')
+ return result
+ } catch (err) {
+ try {
+ await client.query('ROLLBACK')
+ } catch (rollbackError) {
+ releaseError = rollbackError instanceof Error ? rollbackError : true
+ }
+ throw err
+ } finally {
+ client.release(releaseError)
+ }
+ }
+}
+
export type CodexDiscoveredModel = {
model: string
displayName?: string
@@ -321,11 +367,21 @@ export async function rebuildLiveCodexUnionAllowlist(db: DbClient): Promise<void
)
}
+/**
+ * Reconcile one connection's catalog. The provider call runs outside any
+ * transaction; the fenced readiness/revision write, the model row
+ * insert/refresh/stale and the live union rebuild commit together, so a
+ * failure in any phase leaves readiness, revision and rows untouched. The
+ * fenced UPDATE row-locks the connection first, so concurrent syncs on one
+ * revision serialize and exactly one commits an outcome. Callers publish the
+ * runtime allowlist only after this resolves (i.e. after commit).
+ */
export async function syncCodexSubscriptionCatalog(
db: DbClient,
transport: CodexCatalogTransport,
accessToken: string,
- expected?: { credentialRevision?: number; catalogRevision?: number; connectionKey?: string }
+ expected?: { credentialRevision?: number; catalogRevision?: number; connectionKey?: string },
+ options: { withTransaction?: CodexTransactionRunner } = {}
): Promise<{
outcome: CodexCatalogOutcome
connection: CodexSubscriptionSafeConnection | null
@@ -340,25 +396,28 @@ export async function syncCodexSubscriptionCatalog(
}
const expectedCredentialRevision = expected?.credentialRevision ?? connection.credentialRevision
const expectedCatalogRevision = expected?.catalogRevision ?? connection.catalogRevision
- const result = await transport.listModels({ accessToken })
- const existing = await loadCodexRows(db, connection.id)
- const plan = planCodexCatalogReconcile(existing, result)
- const recorded = await recordCodexCatalogOutcome(db, {
- catalogStatus: plan.catalogStatus as CodexSubscriptionCatalogStatus,
- connectionStatus: plan.connectionStatus,
- expectedCredentialRevision,
- expectedCatalogRevision,
- connectionKey,
+ const result = boundCodexCatalogResult(await transport.listModels({ accessToken }))
+ const outcomePlan = planCodexCatalogReconcile([], result)
+ const runInTransaction = options.withTransaction ?? codexTransactionRunnerFor(db)
+ const committed = await runInTransaction(async tx => {
+ const recorded = await recordCodexCatalogOutcome(tx, {
+ catalogStatus: outcomePlan.catalogStatus as CodexSubscriptionCatalogStatus,
+ connectionStatus: outcomePlan.connectionStatus,
+ expectedCredentialRevision,
+ expectedCatalogRevision,
+ connectionKey,
+ })
+ if (!recorded || !outcomePlan.mutateRows) {
+ return { recorded, added: 0, refreshed: 0, staled: 0 }
+ }
+ const plan = planCodexCatalogReconcile(await loadCodexRows(tx, connection.id), result)
+ const added = await insertDiscovered(tx, connection.id, plan.inserts)
+ const refreshed = await refreshDiscovered(tx, connection.id, plan.refresh)
+ const staled = await staleMissing(tx, connection.id, plan.stale)
+ await rebuildLiveCodexUnionAllowlist(tx)
+ return { recorded, added, refreshed, staled }
})
- let added = 0
- let refreshed = 0
- let staled = 0
- if (recorded && plan.mutateRows) {
- added = await insertDiscovered(db, connection.id, plan.inserts)
- refreshed = await refreshDiscovered(db, connection.id, plan.refresh)
- staled = await staleMissing(db, connection.id, plan.stale)
- await rebuildLiveCodexUnionAllowlist(db)
- }
+ const { recorded, added, refreshed, staled } = committed
if (!recorded) {
log.warn(
{ event: 'codex_catalog_stale_writer' },
@@ -368,7 +427,7 @@ export async function syncCodexSubscriptionCatalog(
log.info(
{
event: 'codex_catalog_reconciled',
- outcome: plan.catalogStatus,
+ outcome: outcomePlan.catalogStatus,
added,
refreshed,
staled,
@@ -377,7 +436,7 @@ export async function syncCodexSubscriptionCatalog(
)
}
return {
- outcome: plan.catalogStatus,
+ outcome: outcomePlan.catalogStatus,
connection: recorded,
added,
refreshed,
@@ -385,6 +444,24 @@ export async function syncCodexSubscriptionCatalog(
}
}
+function boundCodexCatalogResult(result: CodexCatalogTransportResult): CodexCatalogTransportResult {
+ if (result.outcome !== 'ready') return result
+ const bounded = boundDiscoveredCatalogModels(result.models)
+ if (bounded.droppedInvalidId > 0 || bounded.droppedOverCount > 0) {
+ log.warn(
+ {
+ event: 'codex_catalog_discovery_bounded',
+ discovered: result.models.length,
+ kept: bounded.models.length,
+ droppedInvalidId: bounded.droppedInvalidId,
+ droppedOverCount: bounded.droppedOverCount,
+ },
+ 'Codex catalog discovery exceeded bounds; extra models were ignored'
+ )
+ }
+ return { outcome: 'ready', models: bounded.models }
+}
+
export function createUnavailableCodexCatalogTransport(): CodexCatalogTransport {
return {
async listModels() {
diff --git a/control-api/src/services/codexSubscriptionConnection.ts b/control-api/src/services/codexSubscriptionConnection.ts
index 2eebc6a7e..81abdb6dd 100644
--- a/control-api/src/services/codexSubscriptionConnection.ts
+++ b/control-api/src/services/codexSubscriptionConnection.ts
@@ -444,41 +444,91 @@ export async function rotateCodexSubscriptionCredentials(
}
}
+/**
+ * Revoke the live grant for a key and invalidate every pending OAuth state
+ * (browser and device) for that key in ONE statement: data-modifying CTEs share
+ * a snapshot and commit atomically, so no earlier OAuth work can
+ * complete against the revoked grant. Pending states are cancelled even when
+ * the key has no live row.
+ *
+ * Codex keys are NOT terminal (0105 made them reusable): an OAuth flow started
+ * after this revoke may reconnect the same key as a fresh live row. A
+ * completion that consumed its state before the revoke is rejected by
+ * `wasCodexConnectionRevokedSinceOAuthState`.
+ */
export async function revokeCodexSubscriptionConnection(
db: DbClient,
connectionKey: string = CODEX_SUBSCRIPTION_CONNECTION_KEY
): Promise<CodexSubscriptionSafeConnection | null> {
const key = normalizeCodexConnectionKey(connectionKey)
const result = await db.query(
- `UPDATE codex_subscription_connections
- SET status = 'revoked',
- refresh_token_encrypted = NULL,
- access_token_encrypted = NULL,
- access_token_expires_at = NULL,
- chatgpt_account_id_encrypted = NULL,
- catalog_status = 'never_synced',
- credential_revision = credential_revision + 1,
- refresh_lock_token = NULL,
- refresh_lock_expires_at = NULL,
- revoked_at = now(),
- updated_at = now()
- WHERE connection_key = $1
- AND revoked_at IS NULL
- RETURNING ${SAFE_CONNECTION_COLUMNS}`,
+ `WITH cancelled_states AS (
+ UPDATE codex_subscription_oauth_states
+ SET status = 'cancelled',
+ cancelled_at = now()
+ WHERE connection_key = $1
+ AND status = 'pending'
+ RETURNING 1
+ ),
+ revoked AS (
+ UPDATE codex_subscription_connections
+ SET status = 'revoked',
+ refresh_token_encrypted = NULL,
+ access_token_encrypted = NULL,
+ access_token_expires_at = NULL,
+ chatgpt_account_id_encrypted = NULL,
+ catalog_status = 'never_synced',
+ credential_revision = credential_revision + 1,
+ refresh_lock_token = NULL,
+ refresh_lock_expires_at = NULL,
+ revoked_at = now(),
+ updated_at = now()
+ WHERE connection_key = $1
+ AND revoked_at IS NULL
+ RETURNING ${SAFE_CONNECTION_COLUMNS}
+ ),
+ disabled_models AS (
+ UPDATE codex_catalog_models
+ SET enabled = false,
+ stale = true
+ WHERE connection_id IN (SELECT id FROM revoked)
+ RETURNING 1
+ )
+ SELECT ${SAFE_CONNECTION_COLUMNS}
+ FROM revoked`,
[key]
)
const row = result.rows[0] as SafeConnectionRow | undefined
if (!row) return getSafeCodexSubscriptionConnection(db, key)
- await db.query(
- `UPDATE codex_catalog_models
- SET enabled = false,
- stale = true
- WHERE connection_id = $1`,
- [row.id]
- )
return toSafeConnection(row)
}
+/**
+ * True when a grant row for `connectionKey` was revoked at or after the OAuth
+ * `state` was created, i.e. that OAuth work predates a revoke of that
+ * key. Compared inside PostgreSQL so both timestamps keep microsecond
+ * precision. A flow started after the revoke is not affected.
+ */
+export async function wasCodexConnectionRevokedSinceOAuthState(
+ db: DbClient,
+ connectionKey: string,
+ state: string
+): Promise<boolean> {
+ const result = await db.query(
+ `SELECT EXISTS (
+ SELECT 1
+ FROM codex_subscription_connections c
+ JOIN codex_subscription_oauth_states s ON s.state = $2
+ WHERE c.connection_key = $1
+ AND c.revoked_at IS NOT NULL
+ AND c.revoked_at >= s.created_at
+ ) AS revoked_since`,
+ [normalizeCodexConnectionKey(connectionKey), state]
+ )
+ const row = result.rows[0] as { revoked_since?: boolean } | undefined
+ return row?.revoked_since === true
+}
+
export async function acquireCodexSubscriptionRefreshLock(
db: DbClient,
lockToken: string,
@@ -658,6 +708,18 @@ export async function recordCodexCatalogOutcome(
return row ? toSafeConnection(row) : null
}
+/**
+ * A first-grant INSERT lost the live-key uniqueness race: another writer
+ * created a live row for the same key first. Callers map it to stale_revision.
+ */
+export function isCodexConnectionKeyConflict(err: unknown): boolean {
+ const conflict = err as { code?: string; constraint?: string } | null
+ return (
+ conflict?.code === '23505' &&
+ conflict.constraint === 'codex_subscription_connections_active_key'
+ )
+}
+
function remapFingerprintConflict(err: unknown): never | Error {
const conflict = err as { code?: string; constraint?: string } | null
if (
diff --git a/control-api/src/services/codexSubscriptionOAuth.ts b/control-api/src/services/codexSubscriptionOAuth.ts
index fff3b460e..a3ff80f35 100644
--- a/control-api/src/services/codexSubscriptionOAuth.ts
+++ b/control-api/src/services/codexSubscriptionOAuth.ts
@@ -15,6 +15,7 @@ import {
acquireCodexSubscriptionRefreshLock,
getSafeCodexSubscriptionConnection,
insertInitialCodexSubscriptionConnection,
+ isCodexConnectionKeyConflict,
loadCodexSubscriptionSecrets,
normalizeCodexConnectionKey,
persistCodexChatgptAccountId,
@@ -22,6 +23,7 @@ import {
revokeCodexSubscriptionConnection,
rotateCodexSubscriptionCredentials,
updateCodexAccessTokenInPlace,
+ wasCodexConnectionRevokedSinceOAuthState,
} from './codexSubscriptionConnection.js'
import {
type CodexSubscriptionOAuthIntent,
@@ -292,7 +294,8 @@ export async function handleCodexBrowserCallback(
deps,
consumed.safe.intent,
token,
- completionConnectionKey(deps, consumed.safe.connectionKey)
+ completionConnectionKey(deps, consumed.safe.connectionKey),
+ input.state
)
}
@@ -347,7 +350,8 @@ export async function pollCodexDevice(
deps,
consumed.safe.intent,
tokenResult.parsed,
- completionConnectionKey(deps, consumed.safe.connectionKey)
+ completionConnectionKey(deps, consumed.safe.connectionKey),
+ state
)
return { status: 'connected', connection }
}
@@ -431,13 +435,49 @@ export async function revokeCodexSubscription(
return local ?? { connectionKey: key, status: 'disconnected' }
}
+function revokedDuringAuthorizationError(): CodexSubscriptionOAuthError {
+ return new CodexSubscriptionOAuthError(
+ 'state_cancelled',
+ 'connection was revoked after this authorization started; start a new connection flow'
+ )
+}
+
+/**
+ * A completion lost a write race on its key. If a revoke landed after this
+ * flow's state was created, the authorization was invalidated
+ * (state_cancelled); otherwise another writer holds the live row
+ * (stale_revision, 409).
+ */
+async function lostKeyRaceError(
+ deps: CodexOAuthDeps,
+ key: string,
+ state: string,
+ message: string
+): Promise<CodexSubscriptionOAuthError> {
+ if (await wasCodexConnectionRevokedSinceOAuthState(deps.db, key, state)) {
+ return revokedDuringAuthorizationError()
+ }
+ return new CodexSubscriptionOAuthError('stale_revision', message)
+}
+
async function persistGrantedTokens(
deps: CodexOAuthDeps,
intent: CodexSubscriptionOAuthIntent,
parsed: ParsedCodexToken,
- connectionKey: string
+ connectionKey: string,
+ state: string
): Promise<CodexSubscriptionSafeConnection> {
const key = normalizeCodexConnectionKey(connectionKey)
+ // Revoke cancels pending states, but a completion may already have consumed
+ // its state when the revoke landed. That authorization predates the revoke
+ // and must not recreate the grant; a flow started after the revoke may.
+ if (await wasCodexConnectionRevokedSinceOAuthState(deps.db, key, state)) {
+ log.warn(
+ { event: 'codex_oauth_completion_revoked', connectionKey: key },
+ 'OAuth completion predates a revoke of its connection'
+ )
+ throw revokedDuringAuthorizationError()
+ }
const existing = await getSafeCodexSubscriptionConnection(deps.db, key)
if (existing?.revokedAt || existing?.status === 'revoked') {
throw new CodexSubscriptionOAuthError(
@@ -470,14 +510,27 @@ async function persistGrantedTokens(
)
}
if (!existing) {
- const created = await insertInitialCodexSubscriptionConnection(
- deps.db,
- deps.encryptionKey,
- write,
- key
- )
- log.info({ event: 'codex_oauth_persisted', connectionKey: key }, 'Codex grant persisted')
- return created
+ try {
+ const created = await insertInitialCodexSubscriptionConnection(
+ deps.db,
+ deps.encryptionKey,
+ write,
+ key
+ )
+ log.info({ event: 'codex_oauth_persisted', connectionKey: key }, 'Codex grant persisted')
+ return created
+ } catch (err) {
+ if (isCodexConnectionKeyConflict(err)) {
+ throw await lostKeyRaceError(deps, key, state, 'a concurrent grant created this connection')
+ }
+ if (err instanceof CodexSubscriptionFingerprintConflictError) {
+ throw new CodexSubscriptionOAuthError(
+ 'fingerprint_in_use',
+ 'a live Codex subscription already uses this ChatGPT account'
+ )
+ }
+ throw err
+ }
}
try {
const rotated = await rotateCodexSubscriptionCredentials(
@@ -491,10 +544,7 @@ async function persistGrantedTokens(
return rotated
} catch (err) {
if (err instanceof CodexSubscriptionStaleRevisionError) {
- throw new CodexSubscriptionOAuthError(
- 'stale_revision',
- 'connection was replaced concurrently'
- )
+ throw await lostKeyRaceError(deps, key, state, 'connection was replaced concurrently')
}
if (err instanceof CodexSubscriptionFingerprintConflictError) {
throw new CodexSubscriptionOAuthError(
diff --git a/control-api/src/services/grokSubscriptionCatalog.ts b/control-api/src/services/grokSubscriptionCatalog.ts
index 4d4adb23d..48a8634a2 100644
--- a/control-api/src/services/grokSubscriptionCatalog.ts
+++ b/control-api/src/services/grokSubscriptionCatalog.ts
@@ -12,6 +12,7 @@ import {
readHostGrokConnectionRef,
recordGrokCatalogOutcome,
} from './grokSubscriptionConnection.js'
+import { boundDiscoveredCatalogModels } from './subscriptionCatalogBounds.js'
const log = rootLogger.child({ module: 'grok-subscription-catalog' })
const PROVIDER = 'grok-subscription'
@@ -314,7 +315,7 @@ export async function syncGrokSubscriptionCatalog(
}
const expectedCredentialRevision = expected.credentialRevision ?? connection.credentialRevision
const expectedCatalogRevision = expected.catalogRevision ?? connection.catalogRevision
- const result = await transport.listModels({ accessToken })
+ const result = boundGrokCatalogResult(await transport.listModels({ accessToken }))
const outcomePlan = planGrokCatalogReconcile([], result)
const runInTransaction = options.withTransaction ?? defaultGrokTransactionRunner
const committed = await runInTransaction(async tx => {
@@ -362,6 +363,24 @@ export async function syncGrokSubscriptionCatalog(
}
}
+function boundGrokCatalogResult(result: GrokCatalogTransportResult): GrokCatalogTransportResult {
+ if (result.outcome !== 'ready') return result
+ const bounded = boundDiscoveredCatalogModels(result.models)
+ if (bounded.droppedInvalidId > 0 || bounded.droppedOverCount > 0) {
+ log.warn(
+ {
+ event: 'grok_catalog_discovery_bounded',
+ discovered: result.models.length,
+ kept: bounded.models.length,
+ droppedInvalidId: bounded.droppedInvalidId,
+ droppedOverCount: bounded.droppedOverCount,
+ },
+ 'Grok catalog discovery exceeded bounds; extra models were ignored'
+ )
+ }
+ return { outcome: 'ready', models: bounded.models }
+}
+
export function createUnavailableGrokCatalogTransport(): GrokCatalogTransport {
return {
async listModels() {
diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.ts
index 2b734b9f5..4df8573d5 100644
--- a/control-api/src/services/llmProviderAttemptStore.ts
+++ b/control-api/src/services/llmProviderAttemptStore.ts
@@ -1,4 +1,7 @@
import type { DbClient } from '../db.js'
+import { rootLogger } from '../observability/logger.js'
+
+const log = rootLogger.child({ module: 'llm-provider-attempt-store' })
export const LLM_PROVIDER_ATTEMPT_PROVIDER = 'codex-subscription' as const
export const GROK_PROVIDER_ATTEMPT_PROVIDER = 'grok-subscription' as const
@@ -206,6 +209,106 @@ export async function applyLlmProviderAttemptSdkLinkOnDeleteSetNullSchema(
`)
}
+/**
+ * 0114 — restore `llm_provider_attempts.connection_id` integrity.
+ *
+ * 0103 added `connection_id REFERENCES codex_subscription_connections(id)`;
+ * 0112 dropped that FK so Grok attempts could reference
+ * `grok_subscription_connections` and did not replace it. A plain FK cannot
+ * point at one of two tables by provider, so a CONSTRAINT TRIGGER enforces the
+ * same invariant on every INSERT and on UPDATEs that touch `provider` or
+ * `connection_id`: a non-null connection_id must exist in the provider's own
+ * connection table (codex-subscription -> codex_subscription_connections,
+ * grok-subscription -> grok_subscription_connections). A Codex attempt may
+ * still carry NULL (pre-0103 rows, old pods); Grok NULL is already rejected by
+ * the 0112 CHECK. Unknown providers with a connection id fail closed.
+ *
+ * Connection rows are never deleted (revocation tombstones them; DELETE is
+ * revoked from control_api_runtime), so an existence check at write time is
+ * equivalent to the dropped FK for runtime writers.
+ *
+ * NOT VALID semantics: triggers never re-check existing rows, so historic
+ * dangling rows cannot fail this migration and stay readable/finalizable
+ * (status/outcome updates do not fire the trigger). The count of such rows is
+ * logged (count only). Idempotent (CREATE OR REPLACE + DROP
+ * TRIGGER IF EXISTS) and old-pod compatible: pods only ever write connection
+ * ids they just read from the provider's table.
+ */
+export async function applyLlmProviderAttemptConnectionIntegritySchema(
+ db: DbClient
+): Promise<void> {
+ await db.query(`
+ CREATE OR REPLACE FUNCTION llm_provider_attempts_assert_connection_exists()
+ RETURNS TRIGGER
+ LANGUAGE plpgsql
+ SET search_path = pg_catalog, public
+ AS $$
+ DECLARE
+ connection_exists BOOLEAN;
+ BEGIN
+ IF NEW.connection_id IS NULL THEN
+ RETURN NULL;
+ END IF;
+ IF NEW.provider = 'codex-subscription' THEN
+ SELECT EXISTS (
+ SELECT 1 FROM public.codex_subscription_connections WHERE id = NEW.connection_id
+ ) INTO connection_exists;
+ ELSIF NEW.provider = 'grok-subscription' THEN
+ SELECT EXISTS (
+ SELECT 1 FROM public.grok_subscription_connections WHERE id = NEW.connection_id
+ ) INTO connection_exists;
+ ELSE
+ connection_exists := false;
+ END IF;
+ IF NOT connection_exists THEN
+ RAISE EXCEPTION USING
+ ERRCODE = 'foreign_key_violation',
+ CONSTRAINT = 'llm_provider_attempts_connection_integrity',
+ MESSAGE = format(
+ 'llm_provider_attempts.connection_id is not a %s connection',
+ NEW.provider
+ );
+ END IF;
+ RETURN NULL;
+ END;
+ $$;
+
+ REVOKE ALL ON FUNCTION llm_provider_attempts_assert_connection_exists() FROM PUBLIC;
+
+ DROP TRIGGER IF EXISTS llm_provider_attempts_connection_integrity ON llm_provider_attempts;
+
+ CREATE CONSTRAINT TRIGGER llm_provider_attempts_connection_integrity
+ AFTER INSERT OR UPDATE OF provider, connection_id ON llm_provider_attempts
+ FOR EACH ROW
+ EXECUTE FUNCTION llm_provider_attempts_assert_connection_exists();
+ `)
+ const danglingAttempts = await countDanglingLlmProviderAttemptConnections(db)
+ if (danglingAttempts > 0) {
+ log.warn(
+ { event: 'llm_provider_attempts_dangling_connection_ids', danglingAttempts },
+ 'historic provider attempts reference a missing connection; new writes are enforced'
+ )
+ }
+}
+
+/** Attempts whose non-null connection_id is missing from the provider's connection table. */
+export async function countDanglingLlmProviderAttemptConnections(db: DbClient): Promise<number> {
+ const result = await db.query(`
+ SELECT COUNT(*)::int AS dangling
+ FROM llm_provider_attempts a
+ WHERE a.connection_id IS NOT NULL
+ AND NOT (
+ (a.provider = 'codex-subscription' AND EXISTS (
+ SELECT 1 FROM codex_subscription_connections c WHERE c.id = a.connection_id
+ ))
+ OR (a.provider = 'grok-subscription' AND EXISTS (
+ SELECT 1 FROM grok_subscription_connections g WHERE g.id = a.connection_id
+ ))
+ )
+ `)
+ return Number((result.rows[0] as { dangling?: number } | undefined)?.dangling ?? 0)
+}
+
export async function applyLlmProviderAttemptTicketSchema(db: DbClient): Promise<void> {
await db.query(`
CREATE TABLE IF NOT EXISTS llm_provider_attempt_tickets (
diff --git a/control-api/src/services/subscriptionCatalogBounds.ts b/control-api/src/services/subscriptionCatalogBounds.ts
new file mode 100644
index 000000000..d630bb5e8
--- /dev/null
+++ b/control-api/src/services/subscriptionCatalogBounds.ts
@@ -0,0 +1,49 @@
+/**
+ * Bounds applied to a subscription provider's discovered model catalog before
+ * it is reconciled into `codex_catalog_models` / `grok_catalog_models`. The
+ * upstream list is provider-controlled; without a bound one sync could insert
+ * an unbounded number of enabled rows (and republish them in the runtime
+ * allowlist ConfigMap). Shared by the Codex and Grok catalog services so both
+ * brokers enforce the same limits (the proxies mirror them).
+ */
+export const MAX_DISCOVERED_CATALOG_MODELS = 256
+export const MAX_CATALOG_MODEL_ID_LENGTH = 128
+
+export type BoundedDiscoveredCatalogModels<T> = {
+ models: T[]
+ /** Entries whose id is not a non-empty string of at most 128 characters. */
+ droppedInvalidId: number
+ /** Entries whose (valid) id falls beyond the first 256 distinct ids. */
+ droppedOverCount: number
+}
+
+/**
+ * Keeps entries with a valid model id whose id is among the first
+ * `MAX_DISCOVERED_CATALOG_MODELS` distinct ids, in provider order. Duplicate
+ * ids of a kept model stay in the list so reconcile keeps its existing
+ * last-entry-wins metadata semantics.
+ */
+export function boundDiscoveredCatalogModels<T extends { model: unknown }>(
+ models: readonly T[]
+): BoundedDiscoveredCatalogModels<T> {
+ const kept: T[] = []
+ const distinct = new Set<string>()
+ let droppedInvalidId = 0
+ let droppedOverCount = 0
+ for (const entry of models) {
+ const id = entry?.model
+ if (typeof id !== 'string' || id.length === 0 || id.length > MAX_CATALOG_MODEL_ID_LENGTH) {
+ droppedInvalidId += 1
+ continue
+ }
+ if (!distinct.has(id)) {
+ if (distinct.size >= MAX_DISCOVERED_CATALOG_MODELS) {
+ droppedOverCount += 1
+ continue
+ }
+ distinct.add(id)
+ }
+ kept.push(entry)
+ }
+ return { models: kept, droppedInvalidId, droppedOverCount }
+}
diff --git a/control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts b/control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts
new file mode 100644
index 000000000..bb915dd36
--- /dev/null
+++ b/control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts
@@ -0,0 +1,196 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { Pool } from 'pg'
+import { initDb } from '../src/db.js'
+import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
+import { insertInitialCodexSubscriptionConnection } from '../src/services/codexSubscriptionConnection.js'
+import { insertInitialGrokSubscriptionConnection } from '../src/services/grokSubscriptionConnection.js'
+import {
+ type LlmProviderAttemptInsert,
+ applyLlmProviderAttemptConnectionIntegritySchema,
+ countDanglingLlmProviderAttemptConnections,
+ insertLlmProviderAttempt,
+} from '../src/services/llmProviderAttemptStore.js'
+import './realPostgres.requirement.ts'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+function attempt(overrides: Partial<LlmProviderAttemptInsert>): LlmProviderAttemptInsert {
+ return {
+ callerKind: 'host',
+ hostRef: 'research-host',
+ invocationId: `invocation-${randomBytes(8).toString('hex')}`,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ model: 'gpt-5.1',
+ requestHash: 'd'.repeat(64),
+ policyRevision: 1,
+ policyHash: 'e'.repeat(64),
+ budgetReservationId: 'unbudgeted',
+ connectionRevision: 1,
+ ...overrides,
+ }
+}
+
+describeRealPostgres('0114 llm_provider_attempts connection integrity on real PostgreSQL', () => {
+ const database = `attempt_integrity_${randomBytes(6).toString('hex')}`
+ let adminPool: Pool
+ let pool: Pool
+ let codexConnectionId: string
+ let grokConnectionId: string
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ pool = new Pool({ connectionString: databaseUrl(adminUrl!, database) })
+ await initDb({ connect: () => pool.connect() })
+ codexConnectionId = (
+ await insertInitialCodexSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-codex', accountFingerprint: 'fp-codex-integrity' },
+ 'team-codex-integrity'
+ )
+ ).id
+ grokConnectionId = (
+ await insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-grok', accountFingerprint: 'fp-grok-integrity' },
+ 'team-grok-integrity'
+ )
+ ).id
+ }, 60_000)
+
+ afterAll(async () => {
+ await pool?.end()
+ if (adminPool) {
+ await adminPool
+ .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ .catch(() => undefined)
+ await adminPool.end()
+ }
+ })
+
+ it('accepts attempts whose connection_id exists in the provider table', async () => {
+ const codex = await insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: codexConnectionId })
+ )
+ expect(codex.connectionId).toBe(codexConnectionId)
+ const grok = await insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'grok-subscription', model: 'grok-4.6', connectionId: grokConnectionId })
+ )
+ expect(grok.connectionId).toBe(grokConnectionId)
+ // Pre-0103 Codex rows (and old pods) may still carry no connection id.
+ const legacy = await insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: null })
+ )
+ expect(legacy.connectionId).toBeNull()
+ })
+
+ it('rejects a Codex attempt that names a Grok connection id', async () => {
+ await expect(
+ insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: grokConnectionId })
+ )
+ ).rejects.toMatchObject({
+ code: '23503',
+ constraint: 'llm_provider_attempts_connection_integrity',
+ })
+ })
+
+ it('rejects a Grok attempt with a connection id that exists nowhere or only for Codex', async () => {
+ for (const connectionId of [randomUUID(), codexConnectionId]) {
+ await expect(
+ insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'grok-subscription', model: 'grok-4.6', connectionId })
+ )
+ ).rejects.toMatchObject({
+ code: '23503',
+ constraint: 'llm_provider_attempts_connection_integrity',
+ })
+ }
+ })
+
+ it('rejects a Codex attempt with a random connection id', async () => {
+ await expect(
+ insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: randomUUID() })
+ )
+ ).rejects.toMatchObject({ code: '23503' })
+ })
+
+ it('rejects updates that point connection_id or provider at another table', async () => {
+ const row = await insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: codexConnectionId })
+ )
+ await expect(
+ pool.query(`UPDATE llm_provider_attempts SET connection_id = $1 WHERE id = $2`, [
+ randomUUID(),
+ row.id,
+ ])
+ ).rejects.toMatchObject({ code: '23503' })
+ await expect(
+ pool.query(`UPDATE llm_provider_attempts SET provider = 'grok-subscription' WHERE id = $1`, [
+ row.id,
+ ])
+ ).rejects.toMatchObject({ code: '23503' })
+ // Lifecycle updates that do not touch provider/connection_id still work.
+ await expect(
+ pool.query(`UPDATE llm_provider_attempts SET status = 'redeemed' WHERE id = $1`, [row.id])
+ ).resolves.toMatchObject({ rowCount: 1 })
+ })
+
+ it('re-applies idempotently and never fails on historic dangling rows', async () => {
+ await pool.query(
+ `DROP TRIGGER IF EXISTS llm_provider_attempts_connection_integrity ON llm_provider_attempts`
+ )
+ const dangling = await insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: randomUUID() })
+ )
+ await expect(applyLlmProviderAttemptConnectionIntegritySchema(pool)).resolves.toBeUndefined()
+ await expect(applyLlmProviderAttemptConnectionIntegritySchema(pool)).resolves.toBeUndefined()
+ expect(await countDanglingLlmProviderAttemptConnections(pool)).toBe(1)
+ const triggers = await pool.query<{ tgname: string; tgconstraint: string }>(
+ `SELECT tgname, tgconstraint::text
+ FROM pg_trigger
+ WHERE tgrelid = 'llm_provider_attempts'::regclass
+ AND tgname = 'llm_provider_attempts_connection_integrity'`
+ )
+ expect(triggers.rows).toHaveLength(1)
+ expect(triggers.rows[0]?.tgconstraint).not.toBe('0')
+ // The historic row stays readable and finalizable (status-only update).
+ await expect(
+ pool.query(`UPDATE llm_provider_attempts SET status = 'redeemed' WHERE id = $1`, [
+ dangling.id,
+ ])
+ ).resolves.toMatchObject({ rowCount: 1 })
+ // New writes are enforced again.
+ await expect(
+ insertLlmProviderAttempt(
+ pool,
+ attempt({ provider: 'codex-subscription', connectionId: randomUUID() })
+ )
+ ).rejects.toMatchObject({ code: '23503' })
+ })
+})
diff --git a/control-api/test/db.llmProviderAttemptMigration.test.ts b/control-api/test/db.llmProviderAttemptMigration.test.ts
index b0c8b1571..1db1d0540 100644
--- a/control-api/test/db.llmProviderAttemptMigration.test.ts
+++ b/control-api/test/db.llmProviderAttemptMigration.test.ts
@@ -143,6 +143,30 @@ describe('00a2/00a3 LLM provider-attempt ledger', () => {
expect(storeSource).not.toMatch(/GRANT .* TO .*proxy/i)
})
+ it('registers 0114 connection integrity after 0113 as a provider-aware constraint trigger', () => {
+ const v0113 = dbSource.indexOf("version: '0113_grok_subscription_terminal_connection_key'")
+ const v0114 = dbSource.indexOf("version: '0114_llm_provider_attempts_connection_integrity'")
+ expect(v0113).toBeGreaterThan(-1)
+ expect(v0114).toBeGreaterThan(v0113)
+ expect(dbSource).toContain('apply: applyLlmProviderAttemptConnectionIntegritySchema')
+ const body = storeSource.slice(
+ storeSource.indexOf('export async function applyLlmProviderAttemptConnectionIntegritySchema')
+ )
+ expect(body).toContain(
+ 'CREATE OR REPLACE FUNCTION llm_provider_attempts_assert_connection_exists()'
+ )
+ expect(body).toContain(
+ 'DROP TRIGGER IF EXISTS llm_provider_attempts_connection_integrity ON llm_provider_attempts'
+ )
+ expect(body).toContain('CREATE CONSTRAINT TRIGGER llm_provider_attempts_connection_integrity')
+ expect(body).toContain(
+ 'AFTER INSERT OR UPDATE OF provider, connection_id ON llm_provider_attempts'
+ )
+ expect(body).toContain('public.codex_subscription_connections WHERE id = NEW.connection_id')
+ expect(body).toContain('public.grok_subscription_connections WHERE id = NEW.connection_id')
+ expect(body).not.toMatch(/VALIDATE CONSTRAINT|ALTER TABLE llm_provider_attempts/)
+ })
+
it('classifies the new relations as upsert in the runtime access contract', () => {
const profile = runtimeAccessProfile()
expect(profile.get('llm_provider_attempts')).toBe('upsert')
diff --git a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
index e7b14131d..52ceedd6f 100644
--- a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
+++ b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
@@ -21,7 +21,7 @@ describe('0101 oauth_grants owner generalization migration', () => {
const { CONTROL_API_MIGRATIONS } = await import('../src/db.js')
const versions = CONTROL_API_MIGRATIONS.map(m => m.version)
expect(versions).toContain('0106_oauth_grants_owner_generalization')
- expect(versions.at(-1)).toBe('0113_grok_subscription_terminal_connection_key')
+ expect(versions.at(-1)).toBe('0114_llm_provider_attempts_connection_integrity')
expect(versions.indexOf('0100_seed_minimax_allowed_model')).toBeLessThan(
versions.indexOf('0106_oauth_grants_owner_generalization')
)
diff --git a/control-api/test/services.codexSubscriptionCatalog.test.ts b/control-api/test/services.codexSubscriptionCatalog.test.ts
index 2e111d5a1..339c21787 100644
--- a/control-api/test/services.codexSubscriptionCatalog.test.ts
+++ b/control-api/test/services.codexSubscriptionCatalog.test.ts
@@ -162,7 +162,8 @@ describe('syncCodexSubscriptionCatalog fencing', () => {
},
},
'access-token',
- { credentialRevision: 1, catalogRevision: 1 }
+ { credentialRevision: 1, catalogRevision: 1 },
+ { withTransaction: work => work({ query }) }
)
expect(result.outcome).toBe('unavailable')
expect(result.connection).toBeNull()
@@ -287,3 +288,123 @@ describe('isCodexAssignmentAllowed', () => {
).resolves.toBe(false)
})
})
+
+describe('syncCodexSubscriptionCatalog atomicity and bounds', () => {
+ const CONNECTION_ROW = {
+ id: '22222222-2222-4222-8222-222222222222',
+ connection_key: 'team-plus',
+ display_name: 'team-plus',
+ default_model: null,
+ created_by: null,
+ status: 'connected',
+ credential_revision: 2,
+ catalog_revision: 0,
+ account_fingerprint: 'fp',
+ catalog_status: 'never_synced',
+ catalog_synced_at: null,
+ last_refresh_at: null,
+ last_auth_at: null,
+ refresh_lock_token: null,
+ refresh_lock_expires_at: null,
+ revoked_at: null,
+ created_at: new Date(),
+ updated_at: new Date(),
+ }
+
+ function harness(opts: { failOn?: RegExp } = {}) {
+ let inTransaction = false
+ const writesOutside: string[] = []
+ const txLog: Array<{ sql: string; values?: unknown[] }> = []
+ const outside = {
+ query: vi.fn(async (sql: string) => {
+ if (/^\s*(INSERT|UPDATE)/i.test(sql)) writesOutside.push(sql)
+ if (sql.includes('FROM codex_subscription_connections')) {
+ return { rows: [CONNECTION_ROW], rowCount: 1 }
+ }
+ return { rows: [], rowCount: 0 }
+ }),
+ }
+ const tx = {
+ query: vi.fn(async (sql: string, values?: unknown[]) => {
+ if (!inTransaction) writesOutside.push(sql)
+ txLog.push({ sql, values })
+ if (opts.failOn?.test(sql)) throw new Error('injected failure')
+ if (sql.includes('UPDATE codex_subscription_connections')) {
+ return { rows: [{ ...CONNECTION_ROW, catalog_status: 'ready' }], rowCount: 1 }
+ }
+ if (sql.includes('INSERT INTO codex_catalog_models')) return { rows: [], rowCount: 1 }
+ return { rows: [], rowCount: 0 }
+ }),
+ }
+ const withTransaction = vi.fn(async <T>(work: (db: typeof tx) => Promise<T>) => {
+ inTransaction = true
+ try {
+ return await work(tx)
+ } finally {
+ inTransaction = false
+ }
+ })
+ return { outside, tx, txLog, writesOutside, withTransaction }
+ }
+
+ it('records readiness, reconciles rows and rebuilds the union inside one transaction', async () => {
+ const h = harness()
+ const listModels = vi.fn(async () => {
+ expect(h.withTransaction).not.toHaveBeenCalled()
+ return { outcome: 'ready' as const, models: [{ model: 'gpt-5.1' }] }
+ })
+ const synced = await syncCodexSubscriptionCatalog(
+ h.outside,
+ { listModels },
+ 'access-token',
+ { connectionKey: 'team-plus' },
+ { withTransaction: h.withTransaction as never }
+ )
+ expect(synced.outcome).toBe('ready')
+ expect(synced.added).toBe(1)
+ expect(h.withTransaction).toHaveBeenCalledTimes(1)
+ expect(h.writesOutside).toEqual([])
+ const sqls = h.txLog.map(entry => entry.sql)
+ expect(sqls.some(sql => sql.includes('UPDATE codex_subscription_connections'))).toBe(true)
+ expect(sqls.some(sql => sql.includes('INSERT INTO codex_catalog_models'))).toBe(true)
+ expect(sqls.some(sql => sql.includes('INSERT INTO llm_allowed_models'))).toBe(true)
+ })
+
+ it('propagates a mid-reconcile failure out of the transaction instead of reporting ready', async () => {
+ const h = harness({ failOn: /INSERT INTO codex_catalog_models/ })
+ await expect(
+ syncCodexSubscriptionCatalog(
+ h.outside,
+ { listModels: async () => ({ outcome: 'ready', models: [{ model: 'gpt-5.1' }] }) },
+ 'access-token',
+ { connectionKey: 'team-plus' },
+ { withTransaction: h.withTransaction as never }
+ )
+ ).rejects.toThrow('injected failure')
+ expect(h.writesOutside).toEqual([])
+ expect(h.txLog.some(entry => entry.sql.includes('INSERT INTO llm_allowed_models'))).toBe(false)
+ })
+
+ it('inserts at most 256 discovered models and skips ids longer than 128 characters', async () => {
+ const h = harness()
+ const models = [
+ { model: 'x'.repeat(129) },
+ ...Array.from({ length: 300 }, (_, index) => ({ model: `gpt-cap-${index}` })),
+ ]
+ const synced = await syncCodexSubscriptionCatalog(
+ h.outside,
+ { listModels: async () => ({ outcome: 'ready', models }) },
+ 'access-token',
+ { connectionKey: 'team-plus' },
+ { withTransaction: h.withTransaction as never }
+ )
+ const inserted = h.txLog
+ .filter(entry => entry.sql.includes('INSERT INTO codex_catalog_models'))
+ .map(entry => String(entry.values?.[1]))
+ expect(synced.added).toBe(256)
+ expect(inserted).toHaveLength(256)
+ expect(inserted[0]).toBe('gpt-cap-0')
+ expect(inserted.at(-1)).toBe('gpt-cap-255')
+ expect(inserted.some(model => model.length > 128)).toBe(false)
+ })
+})
diff --git a/control-api/test/services.codexSubscriptionConnection.test.ts b/control-api/test/services.codexSubscriptionConnection.test.ts
index 4777da16d..87f4173a3 100644
--- a/control-api/test/services.codexSubscriptionConnection.test.ts
+++ b/control-api/test/services.codexSubscriptionConnection.test.ts
@@ -8,9 +8,11 @@ import {
generateCodexConnectionKey,
getSafeCodexSubscriptionConnection,
insertInitialCodexSubscriptionConnection,
+ isCodexConnectionKeyConflict,
listLiveCodexSubscriptionConnections,
normalizeCodexConnectionKey,
readHostCodexConnectionRef,
+ revokeCodexSubscriptionConnection,
rotateCodexSubscriptionCredentials,
updateCodexSubscriptionConnectionMetadata,
} from '../src/services/codexSubscriptionConnection.js'
@@ -187,6 +189,37 @@ describe('codex subscription connection repository', () => {
expect(sql).not.toMatch(/revoked_at = NULL/)
})
+ it('revokes the live row and cancels pending OAuth states for the key in one statement', async () => {
+ query.mockResolvedValueOnce({ rows: [], rowCount: 0 }).mockResolvedValueOnce({
+ rows: [],
+ rowCount: 0,
+ })
+ await revokeCodexSubscriptionConnection({ query }, 'team-plus')
+ const [sql, values] = query.mock.calls[0] as [string, unknown[]]
+ expect(sql).toMatch(
+ /UPDATE codex_subscription_oauth_states[\s\S]*SET status = 'cancelled'[\s\S]*connection_key = \$1[\s\S]*status = 'pending'/
+ )
+ expect(sql).toMatch(/UPDATE codex_subscription_connections[\s\S]*revoked_at = now\(\)/)
+ expect(sql).toMatch(/UPDATE codex_catalog_models[\s\S]*enabled = false/)
+ expect(values).toEqual(['team-plus'])
+ })
+
+ it('recognizes only the live-key unique violation as a first-grant key race', () => {
+ expect(
+ isCodexConnectionKeyConflict({
+ code: '23505',
+ constraint: 'codex_subscription_connections_active_key',
+ })
+ ).toBe(true)
+ expect(
+ isCodexConnectionKeyConflict({
+ code: '23505',
+ constraint: 'codex_subscription_connections_active_fingerprint',
+ })
+ ).toBe(false)
+ expect(isCodexConnectionKeyConflict(new Error('boom'))).toBe(false)
+ })
+
it('refuses metadata writes on a revoked grant', async () => {
query.mockResolvedValueOnce({ rows: [], rowCount: 0 })
await expect(
diff --git a/control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts b/control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts
new file mode 100644
index 000000000..0481a2050
--- /dev/null
+++ b/control-api/test/services.codexSubscriptionLifecycle.realPostgres.integration.test.ts
@@ -0,0 +1,333 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+import { type DbClient, initDb } from '../src/db.js'
+import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
+import {
+ type CodexCatalogTransport,
+ syncCodexSubscriptionCatalog,
+} from '../src/services/codexSubscriptionCatalog.js'
+import {
+ getSafeCodexSubscriptionConnection,
+ insertInitialCodexSubscriptionConnection,
+} from '../src/services/codexSubscriptionConnection.js'
+import {
+ CODEX_OAUTH_DEVICE_TOKEN_URL,
+ CODEX_OAUTH_DEVICE_USERCODE_URL,
+ CODEX_OAUTH_REVOKE_URL,
+ CODEX_OAUTH_TOKEN_URL,
+ type CodexOAuthDeps,
+ handleCodexBrowserCallback,
+ pollCodexDevice,
+ revokeCodexSubscription,
+ startCodexBrowserConnect,
+ startCodexDeviceConnect,
+} from '../src/services/codexSubscriptionOAuth.js'
+import './realPostgres.requirement.ts'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+function idTokenFor(subject: string): string {
+ return `hdr.${Buffer.from(JSON.stringify({ sub: subject })).toString('base64url')}.sig`
+}
+
+function jsonResponse(status: number, body: Record<string, unknown>): Response {
+ return { ok: status >= 200 && status < 300, status, json: async () => body } as Response
+}
+
+/**
+ * Simulated ChatGPT device/browser authorization provider. Device polls
+ * complete immediately; the authorization code carries the account subject so
+ * a test chooses which account each flow grants.
+ */
+function simulatedCodexProvider(
+ options: {
+ subjectFor?: (deviceAuthId: string) => string
+ beforeTokenExchange?: () => Promise<void>
+ } = {}
+): typeof fetch {
+ let issued = 0
+ return (async (url: string | URL, init?: RequestInit) => {
+ const target = String(url)
+ if (target === CODEX_OAUTH_DEVICE_USERCODE_URL) {
+ issued += 1
+ return jsonResponse(200, {
+ device_auth_id: `deviceauth-${issued}-${randomBytes(4).toString('hex')}`,
+ user_code: 'ABCD-EFGH',
+ interval: 5,
+ expires_in: 900,
+ })
+ }
+ if (target === CODEX_OAUTH_DEVICE_TOKEN_URL) {
+ const body = JSON.parse(String(init?.body ?? '{}')) as { device_auth_id?: string }
+ const deviceAuthId = body.device_auth_id ?? 'unknown'
+ return jsonResponse(200, {
+ authorization_code: `authz:${options.subjectFor?.(deviceAuthId) ?? `subject-${deviceAuthId}`}`,
+ code_verifier: 'pkce-from-poll',
+ })
+ }
+ if (target === CODEX_OAUTH_TOKEN_URL) {
+ await options.beforeTokenExchange?.()
+ const form = new URLSearchParams(String(init?.body ?? ''))
+ const code = form.get('code') ?? 'authz:subject-browser'
+ const subject = code.startsWith('authz:') ? code.slice('authz:'.length) : code
+ return jsonResponse(200, {
+ access_token: `access-${randomBytes(4).toString('hex')}`,
+ refresh_token: `refresh-${randomBytes(4).toString('hex')}`,
+ expires_in: 3600,
+ id_token: idTokenFor(subject),
+ })
+ }
+ if (target === CODEX_OAUTH_REVOKE_URL) return jsonResponse(200, {})
+ return jsonResponse(404, {})
+ }) as typeof fetch
+}
+
+describeRealPostgres('Codex subscription grant lifecycle on real PostgreSQL', () => {
+ const database = `codex_lifecycle_${randomBytes(6).toString('hex')}`
+ let adminPool: Pool
+ let pool: Pool
+
+ function oauthDeps(
+ connectionKey: string,
+ fetchFn: typeof fetch = simulatedCodexProvider(),
+ db: DbClient = pool
+ ): CodexOAuthDeps {
+ return {
+ db,
+ encryptionKey: KEY,
+ fetchFn,
+ clientId: 'app_test_client',
+ redirectUri: 'https://control.example/api/v1/auth/codex-subscription/callback',
+ enabled: true,
+ connectionKey,
+ }
+ }
+
+ async function rowsForKey(connectionKey: string) {
+ const result = await pool.query<{ id: string; status: string; revoked_at: Date | null }>(
+ `SELECT id, status, revoked_at
+ FROM codex_subscription_connections
+ WHERE connection_key = $1
+ ORDER BY created_at ASC`,
+ [connectionKey]
+ )
+ return result.rows
+ }
+
+ async function stateStatus(state: string): Promise<string | undefined> {
+ const result = await pool.query<{ status: string }>(
+ `SELECT status FROM codex_subscription_oauth_states WHERE state = $1`,
+ [state]
+ )
+ return result.rows[0]?.status
+ }
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ pool = new Pool({ connectionString: databaseUrl(adminUrl!, database) })
+ await initDb({ connect: () => pool.connect() })
+ }, 60_000)
+
+ afterAll(async () => {
+ await pool?.end()
+ if (adminPool) {
+ await adminPool
+ .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ .catch(() => undefined)
+ await adminPool.end()
+ }
+ })
+
+ for (const key of ['deployment-default', 'team-codex-reconnect']) {
+ it(`revoke cancels pending flows for ${key}, and a NEW device flow reconnects the same key`, async () => {
+ const sameAccount = simulatedCodexProvider({ subjectFor: () => `acct-${key}` })
+ const first = await startCodexDeviceConnect(oauthDeps(key, sameAccount), 'connect')
+ await expect(
+ pollCodexDevice(oauthDeps(key, sameAccount), first.state)
+ ).resolves.toMatchObject({ status: 'connected' })
+
+ const pendingDevice = await startCodexDeviceConnect(oauthDeps(key, sameAccount), 'reconnect')
+ const pendingBrowser = await startCodexBrowserConnect(
+ oauthDeps(key, sameAccount),
+ 'reconnect'
+ )
+ const revoked = await revokeCodexSubscription(oauthDeps(key, sameAccount))
+ expect(revoked.status).toBe('revoked')
+ expect(await stateStatus(pendingDevice.state)).toBe('cancelled')
+ expect(await stateStatus(pendingBrowser.state)).toBe('cancelled')
+ await expect(
+ pollCodexDevice(oauthDeps(key, sameAccount), pendingDevice.state)
+ ).rejects.toMatchObject({ code: 'state_replayed' })
+ expect((await rowsForKey(key)).filter(row => row.revoked_at === null)).toHaveLength(0)
+
+ // Codex keys are NOT terminal: a flow started after the revoke reconnects
+ // the same key (and the same ChatGPT account) as a fresh live row.
+ const reconnect = await startCodexDeviceConnect(oauthDeps(key, sameAccount), 'connect')
+ const connected = await pollCodexDevice(oauthDeps(key, sameAccount), reconnect.state)
+ expect(connected).toMatchObject({ status: 'connected' })
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(2)
+ expect(rows[0]).toMatchObject({ status: 'revoked' })
+ expect(rows[1]).toMatchObject({ status: 'connected', revoked_at: null })
+ expect(await getSafeCodexSubscriptionConnection(pool, key)).toMatchObject({
+ id: rows[1]!.id,
+ status: 'connected',
+ })
+ })
+ }
+
+ it('a browser completion whose state predates a revoke cannot revive the key', async () => {
+ const key = 'team-codex-browser-race'
+ await insertInitialCodexSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-browser-race', accountFingerprint: 'fp-browser-race' },
+ key
+ )
+ // The revoke lands after the callback consumed its state and before the
+ // grant is persisted (the provider's token exchange sits in between).
+ const fetchFn = simulatedCodexProvider({
+ beforeTokenExchange: async () => {
+ await revokeCodexSubscription(oauthDeps(key))
+ },
+ })
+ const started = await startCodexBrowserConnect(oauthDeps(key, fetchFn), 'reconnect')
+ await expect(
+ handleCodexBrowserCallback(oauthDeps(key, fetchFn), {
+ code: 'authz:acct-browser-race',
+ state: started.state,
+ })
+ ).rejects.toMatchObject({ code: 'state_cancelled' })
+ expect(await stateStatus(started.state)).toBe('consumed')
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(1)
+ expect(rows[0]).toMatchObject({ status: 'revoked' })
+ expect(await getSafeCodexSubscriptionConnection(pool, key)).toBeNull()
+ })
+
+ it('a device completion that consumed its state before a revoke cannot revive the key', async () => {
+ const key = 'team-codex-device-race'
+ await insertInitialCodexSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-device-race', accountFingerprint: 'fp-device-race' },
+ key
+ )
+ let revokeRan = false
+ const revokeAfterConsume: DbClient = {
+ query: async (text, values) => {
+ const result = await pool.query(text, values)
+ if (!revokeRan && /SET status = 'consumed'/.test(text)) {
+ revokeRan = true
+ await revokeCodexSubscription(oauthDeps(key))
+ }
+ return result
+ },
+ }
+ const started = await startCodexDeviceConnect(oauthDeps(key), 'reconnect')
+ await expect(
+ pollCodexDevice(oauthDeps(key, simulatedCodexProvider(), revokeAfterConsume), started.state)
+ ).rejects.toMatchObject({ code: 'state_cancelled' })
+ expect(revokeRan).toBe(true)
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(1)
+ expect(rows[0]).toMatchObject({ status: 'revoked' })
+ })
+
+ it('parallel first grants for one key: one connects and the loser gets stale_revision', async () => {
+ const key = 'team-codex-first-grant-race'
+ const held: Array<() => void> = []
+ // Hold each first-grant INSERT until both completions reached it, so both
+ // observed "no live row" before either wrote.
+ const gatedDb: DbClient = {
+ query: async (text, values) => {
+ if (text.includes('INSERT INTO codex_subscription_connections')) {
+ await new Promise<void>(resolve => {
+ held.push(resolve)
+ if (held.length === 2) for (const release of held) release()
+ })
+ }
+ return pool.query(text, values)
+ },
+ }
+ const first = await startCodexDeviceConnect(oauthDeps(key), 'connect')
+ const second = await startCodexDeviceConnect(oauthDeps(key), 'connect')
+ const results = await Promise.allSettled([
+ pollCodexDevice(oauthDeps(key, simulatedCodexProvider(), gatedDb), first.state),
+ pollCodexDevice(oauthDeps(key, simulatedCodexProvider(), gatedDb), second.state),
+ ])
+ expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1)
+ const rejected = results.filter(
+ (result): result is PromiseRejectedResult => result.status === 'rejected'
+ )
+ expect(rejected).toHaveLength(1)
+ expect(rejected[0]?.reason).toMatchObject({ code: 'stale_revision' })
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(1)
+ expect(rows[0]).toMatchObject({ status: 'connected', revoked_at: null })
+ })
+
+ it('a catalog failure mid-sync leaves readiness, revision and model rows unchanged', async () => {
+ const key = 'team-codex-catalog'
+ const created = await insertInitialCodexSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-catalog', accessToken: 'access', accountFingerprint: 'fp-catalog' },
+ key
+ )
+ const failing: CodexCatalogTransport = {
+ listModels: async () => ({
+ outcome: 'ready',
+ // INTEGER overflow fails the second model INSERT after readiness was recorded.
+ models: [
+ { model: 'codex-catalog-ok' },
+ { model: 'codex-catalog-bad', contextWindowTokens: 3_000_000_000 },
+ ],
+ }),
+ }
+ await expect(
+ syncCodexSubscriptionCatalog(pool, failing, 'access', { connectionKey: key })
+ ).rejects.toThrow()
+ expect(await getSafeCodexSubscriptionConnection(pool, key)).toMatchObject({
+ catalogStatus: 'never_synced',
+ catalogRevision: created.catalogRevision,
+ catalogSyncedAt: null,
+ })
+ const models = await pool.query(
+ `SELECT model FROM codex_catalog_models WHERE connection_id = $1`,
+ [created.id]
+ )
+ expect(models.rows).toEqual([])
+ const union = await pool.query(
+ `SELECT model FROM llm_allowed_models WHERE provider = 'codex-subscription' AND model = $1`,
+ ['codex-catalog-ok']
+ )
+ expect(union.rows).toEqual([])
+
+ const healthy = await syncCodexSubscriptionCatalog(
+ pool,
+ { listModels: async () => ({ outcome: 'ready', models: [{ model: 'codex-catalog-ok' }] }) },
+ 'access',
+ { connectionKey: key }
+ )
+ expect(healthy.connection).toMatchObject({
+ catalogStatus: 'ready',
+ catalogRevision: created.catalogRevision + 1,
+ })
+ expect(healthy.added).toBe(1)
+ })
+})
diff --git a/control-api/test/services.codexSubscriptionOAuth.test.ts b/control-api/test/services.codexSubscriptionOAuth.test.ts
index fe243e505..71b5be7e9 100644
--- a/control-api/test/services.codexSubscriptionOAuth.test.ts
+++ b/control-api/test/services.codexSubscriptionOAuth.test.ts
@@ -18,6 +18,7 @@ const repos = vi.hoisted(() => ({
revokeConnection: vi.fn(),
updateInPlace: vi.fn(),
persistAccountId: vi.fn(),
+ revokedSince: vi.fn(),
}))
vi.mock('../src/services/codexSubscriptionOAuthState.js', async () => {
@@ -45,6 +46,7 @@ vi.mock('../src/services/codexSubscriptionConnection.js', async () => {
revokeCodexSubscriptionConnection: repos.revokeConnection,
updateCodexAccessTokenInPlace: repos.updateInPlace,
persistCodexChatgptAccountId: repos.persistAccountId,
+ wasCodexConnectionRevokedSinceOAuthState: repos.revokedSince,
}
})
@@ -280,6 +282,92 @@ describe('codex subscription OAuth broker', () => {
expect(repos.insertInitial).not.toHaveBeenCalled()
})
+ it('rejects a completion whose state predates a revoke of its key without writing a grant', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({
+ ok: true,
+ json: async () => ({
+ access_token: 'access-secret',
+ refresh_token: 'refresh-secret',
+ id_token: idTokenFor('acct_raw_123'),
+ }),
+ })
+ repos.consumeState.mockResolvedValue({
+ safe: {
+ state: 'state-before-revoke',
+ flow: 'browser',
+ intent: 'reconnect',
+ status: 'consumed',
+ connectionKey: 'team-plus',
+ expiresAt: new Date(Date.now() + 1000),
+ consumedAt: new Date(),
+ cancelledAt: null,
+ createdAt: new Date(),
+ },
+ pkceVerifier: 'verifier',
+ })
+ // The key has no live row any more (revoked after this flow started).
+ repos.getSafe.mockResolvedValue(null)
+ repos.revokedSince.mockResolvedValue(true)
+ await expect(
+ handleCodexBrowserCallback(deps(fetchFn), { code: 'code-1', state: 'state-before-revoke' })
+ ).rejects.toMatchObject({ code: 'state_cancelled' })
+ expect(repos.revokedSince.mock.calls[0]?.slice(1)).toEqual(['team-plus', 'state-before-revoke'])
+ expect(repos.insertInitial).not.toHaveBeenCalled()
+ expect(repos.rotate).not.toHaveBeenCalled()
+ })
+
+ it('maps a lost first-grant live-key race to stale_revision, or state_cancelled when a revoke won', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({
+ ok: true,
+ json: async () => ({
+ access_token: 'access-secret',
+ refresh_token: 'refresh-secret',
+ id_token: idTokenFor('acct_raw_123'),
+ }),
+ })
+ repos.consumeState.mockResolvedValue({
+ safe: {
+ state: 'state-race',
+ flow: 'browser',
+ intent: 'connect',
+ status: 'consumed',
+ connectionKey: 'team-plus',
+ expiresAt: new Date(Date.now() + 1000),
+ consumedAt: new Date(),
+ cancelledAt: null,
+ createdAt: new Date(),
+ },
+ pkceVerifier: 'verifier',
+ })
+ repos.getSafe.mockResolvedValue(null)
+ repos.insertInitial.mockRejectedValue(
+ Object.assign(new Error('duplicate key value violates unique constraint'), {
+ code: '23505',
+ constraint: 'codex_subscription_connections_active_key',
+ })
+ )
+ repos.revokedSince.mockResolvedValue(false)
+ await expect(
+ handleCodexBrowserCallback(deps(fetchFn), { code: 'code-1', state: 'state-race' })
+ ).rejects.toMatchObject({ name: 'CodexSubscriptionOAuthError', code: 'stale_revision' })
+
+ repos.revokedSince.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ await expect(
+ handleCodexBrowserCallback(deps(fetchFn), { code: 'code-1', state: 'state-race' })
+ ).rejects.toMatchObject({ code: 'state_cancelled' })
+
+ repos.revokedSince.mockResolvedValue(false)
+ repos.insertInitial.mockRejectedValue(
+ Object.assign(new Error('duplicate key value violates unique constraint'), {
+ code: '23505',
+ constraint: 'some_other_unique',
+ })
+ )
+ await expect(
+ handleCodexBrowserCallback(deps(fetchFn), { code: 'code-1', state: 'state-race' })
+ ).rejects.toMatchObject({ code: '23505', constraint: 'some_other_unique' })
+ })
+
it('persists a different account only with explicit replace intent', async () => {
const fetchFn = vi.fn().mockResolvedValue({
ok: true,
diff --git a/control-api/test/services.grokSubscriptionCatalog.test.ts b/control-api/test/services.grokSubscriptionCatalog.test.ts
index b250717c9..57e809cbc 100644
--- a/control-api/test/services.grokSubscriptionCatalog.test.ts
+++ b/control-api/test/services.grokSubscriptionCatalog.test.ts
@@ -114,6 +114,28 @@ describe('grok subscription catalog', () => {
expect(h.txLog.some(sql => sql.includes('INSERT INTO llm_allowed_models'))).toBe(true)
})
+ it('inserts at most 256 discovered models and skips ids longer than 128 characters', async () => {
+ const h = harness()
+ const models = [
+ { model: 'g'.repeat(129) },
+ ...Array.from({ length: 300 }, (_, index) => ({ model: `grok-cap-${index}` })),
+ ]
+ const synced = await syncGrokSubscriptionCatalog(
+ h.outside,
+ { listModels: async () => ({ outcome: 'ready', models }) },
+ 'access-token',
+ { connectionKey: 'team-grok' },
+ { withTransaction: h.withTransaction as never }
+ )
+ const inserted = h.tx.query.mock.calls
+ .filter(([sql]) => String(sql).includes('INSERT INTO grok_catalog_models'))
+ .map(call => String((call as unknown as [string, unknown[]])[1]?.[1]))
+ expect(synced.added).toBe(256)
+ expect(inserted).toHaveLength(256)
+ expect(inserted.at(-1)).toBe('grok-cap-255')
+ expect(inserted.some(model => model.length > 128)).toBe(false)
+ })
+
it('propagates a mid-reconcile failure out of the transaction instead of reporting ready', async () => {
const h = harness({ failOn: /INSERT INTO grok_catalog_models/ })
await expect(
diff --git a/control-api/test/services.subscriptionCatalogBounds.test.ts b/control-api/test/services.subscriptionCatalogBounds.test.ts
new file mode 100644
index 000000000..f9246e6fc
--- /dev/null
+++ b/control-api/test/services.subscriptionCatalogBounds.test.ts
@@ -0,0 +1,40 @@
+import { describe, expect, it } from 'vitest'
+import {
+ MAX_CATALOG_MODEL_ID_LENGTH,
+ MAX_DISCOVERED_CATALOG_MODELS,
+ boundDiscoveredCatalogModels,
+} from '../src/services/subscriptionCatalogBounds.js'
+
+describe('boundDiscoveredCatalogModels', () => {
+ it('uses the shared 256-model and 128-character limits', () => {
+ expect(MAX_DISCOVERED_CATALOG_MODELS).toBe(256)
+ expect(MAX_CATALOG_MODEL_ID_LENGTH).toBe(128)
+ })
+
+ it('passes a small valid catalog through unchanged', () => {
+ const models = [{ model: 'gpt-5.1' }, { model: 'a'.repeat(128) }]
+ expect(boundDiscoveredCatalogModels(models)).toEqual({
+ models,
+ droppedInvalidId: 0,
+ droppedOverCount: 0,
+ })
+ })
+
+ it('drops invalid ids and keeps the first 256 distinct ids in provider order', () => {
+ const models = [
+ { model: 'b'.repeat(129) },
+ { model: '' },
+ { model: 42 as unknown as string },
+ ...Array.from({ length: 260 }, (_, index) => ({ model: `m-${index}` })),
+ { model: 'm-0', displayName: 'duplicate of a kept id' },
+ ]
+ const bounded = boundDiscoveredCatalogModels(models)
+ expect(bounded.droppedInvalidId).toBe(3)
+ expect(bounded.droppedOverCount).toBe(4)
+ expect(new Set(bounded.models.map(entry => entry.model)).size).toBe(256)
+ expect(bounded.models[0]).toEqual({ model: 'm-0' })
+ expect(bounded.models.at(-2)).toEqual({ model: 'm-255' })
+ // A duplicate of a kept id is retained so reconcile keeps last-entry-wins metadata.
+ expect(bounded.models.at(-1)).toEqual({ model: 'm-0', displayName: 'duplicate of a kept id' })
+ })
+})