Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm): publish the runtime allowlist on every reconciliation tick

Alfredo Lopezf603cc927397
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 6ae781fe2..343c05cc9 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.594",
+ "version": "0.1.596",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.594",
+ "version": "0.1.596",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 5b35eb47d..15acddefd 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.594",
+ "version": "0.1.596",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/main.ts b/control-api/src/main.ts
index 4571932f1..d12a0a357 100644
--- a/control-api/src/main.ts
+++ b/control-api/src/main.ts
@@ -126,8 +126,11 @@ async function main(): Promise<void> {
// afterwards stays invisible to the subscription while the API-key provider of
// the SAME vendor picks it up from the discovery sync above. The tick re-runs
// the identical per-connection sync the Hub's manual action drives, and needs
- // the gateway for the same reason: it publishes the allowlist ConfigMap when a
- // connection records an outcome. Per-broker gates still apply inside the tick.
+ // the gateway for the same reason: it publishes the allowlist ConfigMap on
+ // every tick — unconditionally, like `reconcileAllowedModelsConfigMapOnBoot`
+ // above, so a publish that threw converges on the next tick instead of
+ // stranding a stale runtime snapshot. Per-broker gates still apply inside the
+ // tick.
if (config.subscriptionCatalogSyncCronEnabled) {
startSubscriptionCatalogSyncCron(
{ sync: () => reconcileSubscriptionCatalogsFromEnv(gateway.llmAllowedModelsConfigMap()) },
diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.ts
index 9992f09c1..4ba0d6058 100644
--- a/control-api/src/routes/admin/codexSubscription.ts
+++ b/control-api/src/routes/admin/codexSubscription.ts
@@ -77,7 +77,10 @@ import {
runGrokCatalogSync,
startGrokDeviceConnect,
} from '../../services/grokSubscriptionOAuth.js'
-import { publishAllowedModelsConfigMapAfterGrantChange } from '../../services/llmAllowedModelsConfigMap.js'
+import {
+ publishAllowedModelsConfigMapAfterGrantChange,
+ syncOutcomeChangedTheRow,
+} from '../../services/llmAllowedModelsConfigMap.js'
import { K8sConflictError } from '../../services/resourceService.js'
import {
adminCodexReadRateLimits,
@@ -701,6 +704,17 @@ export function createAdminCodexSubscriptionRouter(
})
return
}
+ // The publish comes BEFORE the error responses, because a failed sync
+ // can still have written the connection row — a rejected refresh token
+ // moves it to `reauth_required` and only then throws, which surfaces
+ // here as `never_synced` plus `persisted`. mcp-host and HCC never read
+ // Postgres, so an unpublished write leaves the runtime serving a grant
+ // the control plane already knows is broken, whatever status code this
+ // handler picks afterwards. `syncOutcomeChangedTheRow` is the same rule
+ // the reconciliation cron applies.
+ if (syncOutcomeChangedTheRow(synced)) {
+ if (await publishRuntimeAllowlistOrFail(res)) return
+ }
if (synced.reason === 'no_grant' || synced.reason === 'disabled') {
res.status(404).json({ error: synced.reason })
return
@@ -716,14 +730,6 @@ export function createAdminCodexSubscriptionRouter(
)
return
}
- // A catalogStatus other than `never_synced` means the sync reached xAI
- // and the connection row now carries that outcome, so the runtime
- // snapshot is stale until it is republished — mcp-host and HCC never
- // read Postgres. `never_synced` persisted nothing and needs no write.
- // The Codex branch below publishes before its own 503 for this reason.
- if (synced.catalogStatus !== 'never_synced') {
- if (await publishRuntimeAllowlistOrFail(res)) return
- }
res.status(503).json({ error: 'catalog_sync_failed', outcome: synced.catalogStatus })
return
}
@@ -760,10 +766,21 @@ export function createAdminCodexSubscriptionRouter(
sendOAuthError(res, new CodexSubscriptionOAuthError(synced.reason, synced.reason))
return
}
- if (await publishRuntimeAllowlistOrFail(res)) return
+ // Same rule as the Grok branch above. No Codex error path writes the
+ // connection row — its only status writes are `cancelled` and `revoked`,
+ // both explicit operator actions — so in practice this publishes exactly
+ // when `catalogStatus` recorded something, and skips the write when the
+ // sync recorded nothing.
+ if (syncOutcomeChangedTheRow(synced)) {
+ if (await publishRuntimeAllowlistOrFail(res)) return
+ }
res.status(503).json({
error: 'catalog_sync_failed',
- outcome: synced.catalogStatus === 'never_synced' ? 'unavailable' : synced.catalogStatus,
+ // Report what the sync reported. Rewriting `never_synced` as
+ // `unavailable` maps "nothing was persisted" onto a status that means
+ // "we reached the vendor and it was down", and the Hub renders this
+ // string to the operator verbatim.
+ outcome: synced.catalogStatus,
added: synced.added ?? 0,
refreshed: synced.refreshed ?? 0,
staled: synced.staled ?? 0,
diff --git a/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts b/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts
index 83712cdd1..8285f310f 100644
--- a/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts
+++ b/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts
@@ -6,8 +6,9 @@
* piling up redundant upstream calls. This suite pins the hazards that make the
* difference between a reconciler and an abuse vector: the lock lifecycle, the
* skip rules (a disabled broker, a grant that is not `connected`), failure
- * isolation per connection, and the single ConfigMap publish whose rule is
- * "any RECORDED outcome", not "a ready outcome".
+ * isolation per connection, and the single ConfigMap publish — which runs on
+ * every tick, including one that recorded nothing, because that is the tick
+ * after a publish that threw and the rows it changed are skipped by status.
*
* Deps are injected: no database, no cluster, no upstream. The production
* wiring is asserted where it belongs — the route suite for the endpoint and
@@ -38,6 +39,19 @@ const STALE: SubscriptionCatalogSyncOutcome = {
catalogStatus: 'never_synced',
reason: 'stale_revision',
}
+/**
+ * The write landed and the sync still reports `never_synced`.
+ * `markGrokRefreshSubjectMismatch` sets `status = 'reauth_required'` and THEN
+ * throws, so the catalog was never synced while the CONNECTION row changed.
+ * `llmAllowedModelsConfigMap` maps that status into the ConfigMap, so the
+ * publish is owed — `never_synced` alone cannot decide it.
+ */
+const PERSISTED_REAUTH: SubscriptionCatalogSyncOutcome = {
+ ok: false,
+ catalogStatus: 'never_synced',
+ reason: 'reauth_required',
+ persisted: true,
+}
/**
* A fake pool sharing one advisory-lock state across every client it hands out
@@ -111,6 +125,7 @@ function makeMaterializer(): AllowedModelsConfigMapMaterializer & {
afterEach(() => {
stopSubscriptionCatalogSyncCron()
vi.useRealTimers()
+ vi.unstubAllGlobals()
})
describe('subscription catalog sync tick — advisory lock', () => {
@@ -319,7 +334,7 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {
expect(materializer.materialize).toHaveBeenCalledTimes(1)
})
- it('does not publish when nothing recorded an outcome', async () => {
+ it('publishes although nothing recorded an outcome, so a failed publish converges', async () => {
const codex = makePort(
'codex-subscription',
[
@@ -334,12 +349,74 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {
// Liveness witness: the loop ran and reached the broker — one row was
// skipped by status and the other really was synced and raced — so the
- // absent publish is the rule under test, not an unreached path.
+ // publish under test happened after a tick that wrote nothing, which is
+ // the whole point. This shape is the tick AFTER a publish that threw: the
+ // rows it changed are `reauth_required` by then and get skipped here, so
+ // gating the publish on "something was recorded" would strand the stale
+ // ConfigMap forever.
expect(codex.listConnections).toHaveBeenCalledTimes(1)
expect(codex.syncCatalog).toHaveBeenCalledWith('raced')
expect(result).toMatchObject({ skipped: 1, raced: 1 })
- expect(materializer.materialize).not.toHaveBeenCalled()
- expect(result.published).toBe('skipped')
+ expect(materializer.materialize).toHaveBeenCalledTimes(1)
+ expect(result.published).toBe('published')
+ })
+
+ it('reports skipped when no materializer is wired, without calling one', async () => {
+ const codex = makePort(
+ 'codex-subscription',
+ [{ connectionKey: 'one', status: 'connected' }],
+ async () => READY
+ )
+
+ const result = await reconcileSubscriptionCatalogs({
+ brokers: [codex],
+ materializer: undefined,
+ })
+
+ // Liveness witness for the negative: the sync really ran, so `skipped`
+ // here is the absent writer and not an abandoned tick.
+ expect(codex.syncCatalog).toHaveBeenCalledWith('one')
+ expect(result).toMatchObject({ synced: 1, published: 'skipped' })
+ })
+
+ it('publishes when the sync persisted a status although it reports never_synced', async () => {
+ const grok = makePort(
+ 'grok-subscription',
+ [{ connectionKey: 'rejected', status: 'connected' }],
+ async () => PERSISTED_REAUTH
+ )
+ const materializer = makeMaterializer()
+
+ const result = await reconcileSubscriptionCatalogs({ brokers: [grok], materializer })
+
+ // Liveness witness: the loop reached the broker and ran the sync, so the
+ // publish below is the rule under test rather than an unvisited path.
+ expect(grok.syncCatalog).toHaveBeenCalledWith('rejected')
+ // The row carries `reauth_required` now. Counting this as `failed` and
+ // skipping the publish leaves mcp-host and HCC serving `connected` with the
+ // full model list until an unrelated grant mutation republishes.
+ expect(result).toMatchObject({ degraded: 1, failed: 0, published: 'published' })
+ expect(materializer.materialize).toHaveBeenCalledTimes(1)
+ })
+
+ it('contains a broker whose connection listing fails, so the other broker still runs', async () => {
+ const codex = makePort('codex-subscription', [], async () => READY)
+ codex.listConnections.mockRejectedValueOnce(new Error('connection listing failed'))
+ const grok = makePort(
+ 'grok-subscription',
+ [{ connectionKey: 'team-grok', status: 'connected' }],
+ async () => READY
+ )
+ const materializer = makeMaterializer()
+
+ const result = await reconcileSubscriptionCatalogs({ brokers: [codex, grok], materializer })
+
+ // `listConnections` runs OUTSIDE the per-connection try, so without its own
+ // guard this rejection escapes the broker loop and the whole tick is lost —
+ // including every Grok connection, which has nothing to do with it.
+ expect(codex.listConnections).toHaveBeenCalledTimes(1)
+ expect(result).toMatchObject({ failed: 1, synced: 1, published: 'published' })
+ expect(grok.syncCatalog).toHaveBeenCalledWith('team-grok')
})
it('records the tick as failed when the publish itself fails, without throwing', async () => {
@@ -365,6 +442,25 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {
describe('subscription catalog cron scheduling', () => {
it('runs a first jittered tick and then one per interval, with unref()d handles', async () => {
vi.useFakeTimers()
+ // The title claims unref(), so the test has to check it. An un-unref'd
+ // handle keeps the event loop alive and control-api stops exiting on
+ // SIGTERM — a failure that never shows up in a test that only counts ticks.
+ const unrefed: string[] = []
+ const realSetTimeout = globalThis.setTimeout
+ const realSetInterval = globalThis.setInterval
+ const recordUnref = <T extends { unref: () => T }>(handle: T, label: string): T => {
+ const original = handle.unref.bind(handle)
+ handle.unref = () => {
+ unrefed.push(label)
+ return original()
+ }
+ return handle
+ }
+ vi.stubGlobal('setTimeout', ((fn: () => void, ms?: number) =>
+ recordUnref(realSetTimeout(fn, ms), 'firstRun')) as unknown as typeof setTimeout)
+ vi.stubGlobal('setInterval', ((fn: () => void, ms?: number) =>
+ recordUnref(realSetInterval(fn, ms), 'interval')) as unknown as typeof setInterval)
+
const shared = { held: false }
const calls: string[] = []
const connector = makeLockPool(shared, calls)
@@ -381,6 +477,9 @@ describe('subscription catalog cron scheduling', () => {
await vi.advanceTimersByTimeAsync(5_000)
expect(sync).toHaveBeenCalledTimes(1)
+ // The first-run handle is unref'd before the timer fires; the interval
+ // handle only exists after it.
+ expect(unrefed).toEqual(['firstRun', 'interval'])
await vi.advanceTimersByTimeAsync(60_000)
expect(sync).toHaveBeenCalledTimes(2)
diff --git a/control-api/src/services/grokSubscriptionOAuth.ts b/control-api/src/services/grokSubscriptionOAuth.ts
index 5685bd9c9..4a0387cac 100644
--- a/control-api/src/services/grokSubscriptionOAuth.ts
+++ b/control-api/src/services/grokSubscriptionOAuth.ts
@@ -118,10 +118,24 @@ export function isGrokOAuthErrorCode(value: string): value is GrokOAuthErrorCode
export class GrokSubscriptionOAuthError extends Error {
readonly code: GrokOAuthErrorCode
- constructor(code: GrokOAuthErrorCode, message: string) {
+ /**
+ * True when this throw FOLLOWS a write to the connection row, so the caller
+ * must treat the connection as changed even though the operation failed. A
+ * rejected refresh token is the case that matters: the row is moved to
+ * `reauth_required` and only then does the error propagate, so the runtime
+ * snapshot is stale until it is republished.
+ */
+ readonly persistedConnectionStatus: boolean
+
+ constructor(
+ code: GrokOAuthErrorCode,
+ message: string,
+ opts: { persistedConnectionStatus?: boolean } = {}
+ ) {
super(message)
this.name = 'GrokSubscriptionOAuthError'
this.code = code
+ this.persistedConnectionStatus = opts.persistedConnectionStatus === true
}
}
@@ -205,6 +219,12 @@ export type GrokCatalogSyncResult =
ok: false
catalogStatus: GrokCatalogOutcome | 'never_synced'
reason?: GrokOAuthErrorCode | 'catalog_sync_failed' | 'stale_revision' | 'no_grant'
+ /**
+ * The catalog was not synced but the CONNECTION row was written anyway.
+ * `catalogStatus` describes the catalog, so it cannot answer "did the row
+ * change?" — only this field can.
+ */
+ persisted?: boolean
}
export async function getGrokSubscriptionConnection(
@@ -449,7 +469,9 @@ export async function runGrokCatalogSync(
)
const reason: GrokOAuthErrorCode | 'catalog_sync_failed' =
err instanceof GrokSubscriptionOAuthError ? err.code : 'catalog_sync_failed'
- return { ok: false, catalogStatus: 'never_synced', reason }
+ const persisted =
+ err instanceof GrokSubscriptionOAuthError && err.persistedConnectionStatus === true
+ return { ok: false, catalogStatus: 'never_synced', reason, persisted }
}
}
@@ -687,7 +709,12 @@ async function exchangeRefreshToken(
)
}
await markGrokRefreshSubjectMismatch(deps.db, fence.connectionKey, fence.expectedRevision)
- throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected')
+ // The row is now `reauth_required`. The throw below must not read as
+ // "nothing happened": the ConfigMap carries this status, so whoever
+ // catches this owes a republish.
+ throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
+ persistedConnectionStatus: true,
+ })
}
throw new GrokSubscriptionOAuthError('provider_unavailable', 'refresh token exchange failed')
}
diff --git a/control-api/src/services/llmAllowedModelsConfigMap.ts b/control-api/src/services/llmAllowedModelsConfigMap.ts
index d818842ae..922dcd47c 100644
--- a/control-api/src/services/llmAllowedModelsConfigMap.ts
+++ b/control-api/src/services/llmAllowedModelsConfigMap.ts
@@ -172,6 +172,30 @@ export interface AllowedModelsConfigMapMaterializer {
*
* Missing writer is a no-op so unit tests without a K8s gateway stay local.
*/
+/**
+ * Did this catalog sync change the connection row, and therefore owe a publish?
+ *
+ * The rule belongs here, beside the publish it gates, because it was previously
+ * written inline in three places in three shapes — the cron, the Grok route and
+ * the Codex route — and one of them was wrong.
+ *
+ * `catalogStatus` describes the CATALOG, so it cannot answer this on its own. A
+ * rejected Grok refresh token writes `status = 'reauth_required'` to the row and
+ * only then throws, leaving `catalogStatus: 'never_synced'` on a row that did
+ * change; `mapCodexConnectionStatusForSnapshot` puts that status in the
+ * ConfigMap, so a skipped publish leaves mcp-host and HCC serving a grant the
+ * control plane already knows is broken. `persisted` is the services' explicit
+ * answer for exactly that case.
+ */
+export function syncOutcomeChangedTheRow(outcome: {
+ ok: boolean
+ catalogStatus: string
+ persisted?: boolean
+}): boolean {
+ if (outcome.ok) return true
+ return outcome.persisted === true || outcome.catalogStatus !== 'never_synced'
+}
+
export async function publishAllowedModelsConfigMapAfterGrantChange(
writer: AllowedModelsConfigMapMaterializer | undefined
): Promise<'published' | 'skipped'> {
diff --git a/control-api/src/services/subscriptionCatalogSyncCron.ts b/control-api/src/services/subscriptionCatalogSyncCron.ts
index a821ebd92..2c62335e7 100644
--- a/control-api/src/services/subscriptionCatalogSyncCron.ts
+++ b/control-api/src/services/subscriptionCatalogSyncCron.ts
@@ -48,7 +48,10 @@ import {
} from './grokSubscriptionConnection.js'
import { runGrokCatalogSync } from './grokSubscriptionOAuth.js'
import type { AllowedModelsConfigMapMaterializer } from './llmAllowedModelsConfigMap.js'
-import { publishAllowedModelsConfigMapAfterGrantChange } from './llmAllowedModelsConfigMap.js'
+import {
+ publishAllowedModelsConfigMapAfterGrantChange,
+ syncOutcomeChangedTheRow,
+} from './llmAllowedModelsConfigMap.js'
const log = rootLogger.child({ service: 'subscription_catalog_sync_cron' })
@@ -59,14 +62,19 @@ const CRON_LOCK_KEY_SQL = "hashtext('subscription-catalog-sync-cron-v1')"
export type SubscriptionBroker = 'codex-subscription' | 'grok-subscription'
/**
- * What `runCodexCatalogSync` / `runGrokCatalogSync` return, reduced to the two
- * fields this module decides on. `catalogStatus: 'never_synced'` is the
- * services' own signal that NOTHING was persisted for the connection; any other
- * status means the row now carries that outcome.
+ * What `runCodexCatalogSync` / `runGrokCatalogSync` return, reduced to the
+ * fields this module decides on.
+ *
+ * `catalogStatus` describes the CATALOG, so it cannot answer "did the connection
+ * row change?" on its own: a rejected Grok refresh token writes
+ * `status = 'reauth_required'` and only then throws, leaving the catalog at
+ * `never_synced` on a row that did change. `persisted` is the services' explicit
+ * answer for that case, and `syncOutcomeChangedTheRow` is the single place that
+ * reads both.
*/
export type SubscriptionCatalogSyncOutcome =
| { ok: true; catalogStatus: 'ready' }
- | { ok: false; catalogStatus: string; reason?: string }
+ | { ok: false; catalogStatus: string; reason?: string; persisted?: boolean }
export type SubscriptionConnectionRow = { connectionKey: string; status: string }
@@ -134,7 +142,9 @@ export function filterAddressableGrokConnections<T extends SubscriptionConnectio
* others their sync. The publish, in contrast, is not isolated — a catalog that
* changed in Postgres but not in the ConfigMap is a divergence the runtime hosts
* would serve until the next mutation, so the rejection propagates and the tick
- * reports `errored`.
+ * reports `errored`. It runs on every tick regardless of what the loop recorded,
+ * which is what makes the next tick repair a publish that threw; see the comment
+ * at the publish itself.
*/
export async function reconcileSubscriptionCatalogs(
deps: SubscriptionCatalogReconcileDeps
@@ -153,7 +163,25 @@ export async function reconcileSubscriptionCatalogs(
)
continue
}
- const rows = await port.listConnections()
+ // Outside the per-connection `try` below, so without this guard a single
+ // broker's listing failure would abandon the OTHER broker's connections
+ // too — the tick would throw out of the loop entirely. One failed listing
+ // is one failure, not a lost tick.
+ let rows: SubscriptionConnectionRow[]
+ try {
+ rows = await port.listConnections()
+ } catch (err) {
+ failed += 1
+ log.warn(
+ {
+ event: 'subscription_catalog_sync_listing_failed',
+ broker: port.broker,
+ err: err instanceof Error ? err.message : String(err),
+ },
+ `${port.broker} connection listing failed; continuing with the remaining brokers`
+ )
+ continue
+ }
for (const row of rows) {
// Every narrower rule is wrong: `disconnected` makes the catalog sync
// throw `not_connected`, `connecting` throws `no_grant` inside the
@@ -167,7 +195,7 @@ export async function reconcileSubscriptionCatalogs(
const outcome = await port.syncCatalog(row.connectionKey)
if (outcome.ok) {
synced += 1
- } else if (outcome.catalogStatus !== 'never_synced') {
+ } else if (syncOutcomeChangedTheRow(outcome)) {
degraded += 1
log.info(
{
@@ -175,6 +203,10 @@ export async function reconcileSubscriptionCatalogs(
broker: port.broker,
connectionKey: row.connectionKey,
catalogStatus: outcome.catalogStatus,
+ // Present when the catalog stayed at `never_synced` and the
+ // CONNECTION row is what changed — the reason is then the only
+ // field that says what the row now carries.
+ reason: outcome.reason,
},
`${port.broker} catalog recorded a non-ready outcome`
)
@@ -207,16 +239,25 @@ export async function reconcileSubscriptionCatalogs(
}
}
- // Publish on ANY recorded outcome, ready or not. A tick that moves a grant to
- // `reauth_required` persists that status, the ConfigMap carries it in its
- // annotations, and the materializer's own contract requires the publish for
- // "recorded non-ready outcomes". Publishing only on `ready` would leave the
- // runtime hosts serving a grant the control plane already knows is broken.
- const recorded = synced + degraded
- const published =
- recorded > 0
- ? await publishAllowedModelsConfigMapAfterGrantChange(deps.materializer)
- : 'skipped'
+ // Publish on EVERY tick that ran, not only on one that wrote a row.
+ //
+ // `materialize()` rebuilds the ConfigMap from Postgres — the allowlist plus
+ // both brokers' live connections and their readiness — so the write is
+ // idempotent and publishing unconditionally costs one API call per interval.
+ // That is what lets a publish that threw converge on the next tick with no
+ // state carried across ticks or replicas: the advisory lock is taken per
+ // tick, so a flag set by the replica that failed would be invisible to
+ // whichever replica wins the next one.
+ //
+ // A "publish only when something was recorded" gate cannot do that job. The
+ // rows a failed tick changed are `reauth_required` by the next tick, the
+ // loop above skips them by status, nothing is recorded, and the stale
+ // ConfigMap — where mcp-host and HCC still see a usable grant — survives
+ // until an unrelated mutation or a control-api restart happens to republish.
+ //
+ // `llmAllowedModelsBootReconcile.ts` already writes unconditionally at boot
+ // for the same anti-drift reason; this makes the cron agree with it.
+ const published = await publishAllowedModelsConfigMapAfterGrantChange(deps.materializer)
return { synced, degraded, raced, failed, skipped, published }
}
@@ -231,21 +272,24 @@ function cronDbClient() {
*
* The laziness is load-bearing for Codex: `normalizeControlUiOrigin` throws on a
* malformed, non-http or path-bearing `CONTROL_API_CONTROL_UI_BASE_URL`, and a
- * throw at boot would take control-api down instead of failing one tick. The
- * redirect URI is built only when Codex is enabled, so a bad value cannot cost
- * Grok its reconciliation either. `runCodexCatalogSync` reaches the refresh path
- * only — `exchangeRefreshToken` sends no `redirect_uri` — but a real value is
- * built anyway rather than a placeholder that would be wrong the day the path
- * changes.
+ * throw at boot would take control-api down instead of failing one tick.
+ *
+ * The redirect URI is built inside `syncCatalog`, not here. Building it while
+ * WIRING the ports would throw before the Grok port exists, so a Codex-only
+ * misconfiguration would cost Grok its entire reconciliation — the failure this
+ * function is meant to contain. Inside `syncCatalog` the throw is caught by the
+ * per-connection `try` in `reconcileSubscriptionCatalogs` and counted as one
+ * failed Codex connection, which is what it is.
+ *
+ * `runCodexCatalogSync` reaches the refresh path only — `exchangeRefreshToken`
+ * sends no `redirect_uri` — but a real value is built anyway rather than a
+ * placeholder that would be wrong the day the path changes.
*/
export function createSubscriptionBrokerPorts(): SubscriptionBrokerPort[] {
const db = cronDbClient()
const encryptionKey = deriveOAuthEncryptionKey(config.oauthEncryptionKey)
const codexTransport = createCodexCatalogTransportFromEnv()
- const codexRedirectUri = config.codexSubscriptionEnabled
- ? buildCodexBrowserRedirectUri(resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, undefined))
- : ''
const codex: SubscriptionBrokerPort = {
broker: 'codex-subscription',
enabled: config.codexSubscriptionEnabled,
@@ -257,7 +301,9 @@ export function createSubscriptionBrokerPorts(): SubscriptionBrokerPort[] {
encryptionKey,
fetchFn: fetch,
clientId: config.codexOAuthClientId,
- redirectUri: codexRedirectUri,
+ redirectUri: buildCodexBrowserRedirectUri(
+ resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, undefined)
+ ),
enabled: config.codexSubscriptionEnabled,
connectionKey,
},
diff --git a/control-api/test/config.subscriptionCatalogSyncCron.test.ts b/control-api/test/config.subscriptionCatalogSyncCron.test.ts
new file mode 100644
index 000000000..5c1a612b3
--- /dev/null
+++ b/control-api/test/config.subscriptionCatalogSyncCron.test.ts
@@ -0,0 +1,151 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { readFileSync } from 'node:fs'
+
+/**
+ * Deploy mirror for the subscription catalog reconciliation cron's config keys.
+ *
+ * The cron reads three keys. Two of them were added by this change and the
+ * third is its older sibling, and nothing so far asserts that what
+ * `config.ts` reads is what the cluster actually sets. A key renamed in code
+ * and not in `configmaps.yaml` leaves the cron silently at its code default,
+ * which for `SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED` means permanently off —
+ * a reconciler that never runs and never says so.
+ *
+ * The assertions are NOT "every key equals the code default". Two of these
+ * values are deliberately not the default, and pinning them to the default
+ * would be wrong: `LLM_CATALOG_SYNC_CRON_ENABLED` is deliberately ON in the
+ * cluster while its code default is off. Each key is pinned to the value the
+ * deployment means, with the reason beside it.
+ *
+ * The pattern (`read` + a fail-loud single-match `extractOne`) follows
+ * `config.pluginSdkRateLimits.test.ts`. A key that is absent from the manifest
+ * makes `extractOne` throw rather than skipping the assertion.
+ */
+
+const CRON_KEYS = [
+ 'SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED',
+ 'SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS',
+ 'LLM_CATALOG_SYNC_CRON_ENABLED',
+] as const
+
+type CronKey = (typeof CRON_KEYS)[number]
+
+/** The code floor in `config.ts`; below it control-api refuses to boot. */
+const MIN_INTERVAL_MS = 15 * 60_000
+
+async function loadConfigWith(overrides: Partial<Record<CronKey, string>>) {
+ const originalValues = new Map<string, string | undefined>()
+ for (const key of CRON_KEYS) {
+ originalValues.set(key, process.env[key])
+ delete process.env[key]
+ }
+ Object.assign(process.env, overrides)
+ vi.resetModules()
+ try {
+ const mod = await import('../src/config.js')
+ return mod.config
+ } finally {
+ for (const key of CRON_KEYS) {
+ const value = originalValues.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ }
+}
+
+function read(relativeFromThisFile: string): string {
+ return readFileSync(new URL(relativeFromThisFile, import.meta.url), 'utf-8')
+}
+
+/** Fail-loud single-match extraction — a miss means the key is unregistered. */
+function extractOne(source: string, pattern: RegExp, label: string): string {
+ const match = source.match(pattern)
+ if (!match || match[1] === undefined) {
+ throw new Error(`Could not extract ${label} with ${pattern} — register the key`)
+ }
+ return match[1]
+}
+
+describe('subscription catalog sync cron config', () => {
+ afterEach(() => {
+ vi.resetModules()
+ })
+
+ it('defaults the reconciliation off and the interval to six hours', async () => {
+ const config = await loadConfigWith({})
+
+ expect(config.subscriptionCatalogSyncCronEnabled).toBe(false)
+ expect(config.subscriptionCatalogSyncIntervalMs).toBe(6 * 60 * 60 * 1000)
+ })
+
+ it('honours an interval override at or above the floor', async () => {
+ const config = await loadConfigWith({
+ SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS: String(MIN_INTERVAL_MS),
+ })
+
+ expect(config.subscriptionCatalogSyncIntervalMs).toBe(MIN_INTERVAL_MS)
+ })
+
+ it('refuses to boot on an interval below the floor', async () => {
+ // Fail loud, at import. A silently clamped interval would turn a typo into
+ // one vendor call per connected grant every few seconds.
+ await expect(
+ loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS: String(MIN_INTERVAL_MS - 1) })
+ ).rejects.toThrow(/SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS must be an integer >= 900000/)
+ })
+
+ it('turns the reconciliation on only for the exact string "true"', async () => {
+ // Same exact-token idiom as the discovery cron: `enabled=1` or `yes` must
+ // not silently start issuing vendor calls.
+ expect(
+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: 'true' }))
+ .subscriptionCatalogSyncCronEnabled
+ ).toBe(true)
+ expect(
+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: '1' }))
+ .subscriptionCatalogSyncCronEnabled
+ ).toBe(false)
+ expect(
+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: 'TRUE' }))
+ .subscriptionCatalogSyncCronEnabled
+ ).toBe(false)
+ })
+
+ it('registers all three cron keys in the base ConfigMap with the deployed values', async () => {
+ const config = await loadConfigWith({})
+ const source = read('../../deploy/base/control-plane/configmaps.yaml')
+
+ // Off in the cluster, matching the code default: each tick calls the broker
+ // once per connected grant, so this is turned on per deployment.
+ expect(
+ extractOne(
+ source,
+ /SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED:\s*"([^"]+)"/,
+ 'SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED in deploy/base/control-plane/configmaps.yaml'
+ )
+ ).toBe('false')
+ expect(config.subscriptionCatalogSyncCronEnabled).toBe(false)
+
+ // Declared at the code default so the interval is visible where it is set.
+ const interval = Number(
+ extractOne(
+ source,
+ /SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS:\s*"(\d+)"/,
+ 'SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS in deploy/base/control-plane/configmaps.yaml'
+ )
+ )
+ expect(interval).toBe(config.subscriptionCatalogSyncIntervalMs)
+ expect(interval).toBeGreaterThanOrEqual(MIN_INTERVAL_MS)
+
+ // Deliberately NOT the code default: the discovery sync is one call to
+ // models.dev per tick, so it runs everywhere.
+ expect(
+ extractOne(
+ source,
+ /LLM_CATALOG_SYNC_CRON_ENABLED:\s*"([^"]+)"/,
+ 'LLM_CATALOG_SYNC_CRON_ENABLED in deploy/base/control-plane/configmaps.yaml'
+ )
+ ).toBe('true')
+ expect(config.llmCatalogSyncCronEnabled).toBe(false)
+ })
+})
diff --git a/control-api/test/routes.admin.codexSubscription.test.ts b/control-api/test/routes.admin.codexSubscription.test.ts
index e8a6e9e34..9e4b4b1c6 100644
--- a/control-api/test/routes.admin.codexSubscription.test.ts
+++ b/control-api/test/routes.admin.codexSubscription.test.ts
@@ -439,7 +439,14 @@ describe('admin Codex subscription routes', () => {
expect(res.status).toBe(503)
expect(res.body.error).toBe('catalog_sync_failed')
expect(JSON.stringify(res.body)).not.toContain('proxy down')
- expect(materialize).toHaveBeenCalledTimes(1)
+ // `never_synced` with no `persisted` flag means the row was not written, so
+ // the ConfigMap carries nothing new. Report what the sync reported: calling
+ // this `unavailable` would claim the vendor was reached and was down.
+ expect(res.body.outcome).toBe('never_synced')
+ // Liveness witness: the handler really ran the sync and really took the
+ // failure path, so the absent publish is the rule and not an unreached one.
+ expect(oauth.runCatalogSync).toHaveBeenCalledTimes(1)
+ expect(materialize).not.toHaveBeenCalled()
assertNoLeak(res.body)
})
diff --git a/control-api/test/routes.admin.grokSubscription.test.ts b/control-api/test/routes.admin.grokSubscription.test.ts
index b5aecace1..2d2cfbde6 100644
--- a/control-api/test/routes.admin.grokSubscription.test.ts
+++ b/control-api/test/routes.admin.grokSubscription.test.ts
@@ -571,10 +571,12 @@ describe('admin Grok subscription routes', () => {
expect(materialize).toHaveBeenCalledTimes(1)
})
- // `catalogStatus: 'never_synced'` is the service's own signal that NOTHING
- // was persisted for the connection: no grant, a lost revision fence, a held
- // refresh lock, an OAuth error before the catalog call. There is no new
- // state for the ConfigMap to carry, so the endpoint must not write one.
+ // `catalogStatus: 'never_synced'` WITHOUT `persisted` is the service's own
+ // signal that nothing at all was written for the connection: no grant, a
+ // lost revision fence, a held refresh lock, an OAuth error raised before
+ // anything touched the row. There is no new state for the ConfigMap to
+ // carry, so the endpoint must not write one. `never_synced` on its own does
+ // NOT mean this — see the persisted case below.
it.each([
[{ reason: 'no_grant' }, 404, { error: 'no_grant' }],
[{ reason: 'disabled' }, 404, { error: 'disabled' }],
@@ -603,6 +605,32 @@ describe('admin Grok subscription routes', () => {
}
)
+ // The write landed and the sync still reports `never_synced`.
+ // `markGrokRefreshSubjectMismatch` sets `status = 'reauth_required'` and
+ // THEN throws, so the catalog was never synced while the CONNECTION row
+ // changed. `llmAllowedModelsConfigMap` maps that status into the ConfigMap,
+ // so the publish is owed before the 400 — and it is owed HERE, because no
+ // later reconciliation repairs it: the cron skips a row that is no longer
+ // `connected`.
+ it('publishes a persisted status before the error response', async () => {
+ const materialize = vi.fn(async () => {})
+ grokOAuth.runGrokCatalogSync.mockResolvedValue({
+ ok: false,
+ catalogStatus: 'never_synced',
+ reason: 'reauth_required',
+ persisted: true,
+ })
+ const res = await request(makeApp(makeGateway(materialize))).post(
+ `${GROK}/connections/team-grok/catalog/sync`
+ )
+ expect(res.status).toBe(400)
+ expect(res.body).toEqual({ error: 'reauth_required' })
+ // Liveness witness: the sync ran and failed with this reason, so the
+ // publish below is the rule under test and not an unreached path.
+ expect(grokOAuth.runGrokCatalogSync).toHaveBeenCalledTimes(1)
+ expect(materialize).toHaveBeenCalledTimes(1)
+ })
+
// A non-ready catalogStatus means the OPPOSITE: the sync reached xAI, the
// answer was refused or unavailable, and the connection row now carries
// that outcome. mcp-host and HCC never read Postgres, so withholding the
diff --git a/control-ui/components/CodexSubscriptionHub/index.tsx b/control-ui/components/CodexSubscriptionHub/index.tsx
index 10f295834..f8ee9a0d5 100644
--- a/control-ui/components/CodexSubscriptionHub/index.tsx
+++ b/control-ui/components/CodexSubscriptionHub/index.tsx
@@ -29,6 +29,7 @@ import {
} from '@lib/codexSubscriptionFeature'
import {
GROK_DEVICE_VERIFICATION_ORIGIN,
+ type GrokCatalogSyncView,
type GrokSubscriptionConnectionView,
createGrokSubscriptionConnection,
isAllowedGrokVerificationUri,
@@ -577,17 +578,25 @@ export function CodexSubscriptionHub() {
setBusyKey(hubRowId(row))
setError('')
try {
- const synced = grok
- ? await syncGrokSubscriptionCatalog(row.connectionKey)
- : await syncCodexSubscriptionCatalog(row.connectionKey)
- const models = grok
- ? await listGrokConnectionModels(row.connectionKey)
- : await listCodexConnectionModels(row.connectionKey)
- setEditModels(models)
- if (synced.connection) setEditing(asHubRow(synced.connection, row.broker))
- // A refresh failure after a landed sync is partial — the catalog did
- // change, so it must not surface as "sync failed".
- await load()
+ // Only the sync call itself may report "sync failed". Everything after it
+ // re-reads what the screen shows, and a failure there is partial: the
+ // catalog on the server DID change. Sharing one `try` made a failed
+ // re-read tell the operator their sync had failed, so they would sign in
+ // again to repeat a sync that had already landed.
+ // The narrower of the two views: the Hub reads only `outcome` and
+ // `connection`, and the Codex view satisfies it.
+ let synced: GrokCatalogSyncView
+ try {
+ synced = grok
+ ? await syncGrokSubscriptionCatalog(row.connectionKey)
+ : await syncCodexSubscriptionCatalog(row.connectionKey)
+ } catch (err) {
+ setError(catalogSyncErrorMessage(err, brand))
+ return
+ }
+
+ // The outcome of the sync is reported before the re-read, so the operator
+ // learns it whether or not the refresh below succeeds.
if (synced.outcome === 'ready') {
showToast('Catalog synced', { tone: 'success' })
} else {
@@ -595,8 +604,19 @@ export function CodexSubscriptionHub() {
tone: 'error',
})
}
- } catch (err) {
- setError(catalogSyncErrorMessage(err, brand))
+
+ try {
+ const models = grok
+ ? await listGrokConnectionModels(row.connectionKey)
+ : await listCodexConnectionModels(row.connectionKey)
+ setEditModels(models)
+ if (synced.connection) setEditing(asHubRow(synced.connection, row.broker))
+ await load()
+ } catch {
+ setError(
+ `The ${brand} catalog was synced, but this view could not be refreshed. Reload the page to see it.`
+ )
+ }
} finally {
setBusyKey(null)
}
diff --git a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
index 77211eb34..616c32d7a 100644
--- a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
+++ b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
@@ -930,6 +930,34 @@ describe('CodexSubscriptionHub catalog re-sync', () => {
expect(await screen.findByText('Catalog synced')).toBeInTheDocument()
})
+ // The sync POST landed and the catalog on the server DID change. Only the
+ // re-read of the models failed. Telling the operator "sync failed" here sends
+ // them to sign in again and repeat a sync that already succeeded — and, on a
+ // grant that rotates its refresh token, the repeat has a real cost.
+ it('keeps a landed sync separate from a failed refresh of the view', async () => {
+ vi.mocked(listGrokConnectionModels)
+ .mockResolvedValueOnce([{ model: 'grok-4.6', enabled: true, stale: false }])
+ .mockRejectedValueOnce(new Error('models listing failed'))
+ fetchMock.mockResolvedValueOnce(
+ makeResponse(200, {
+ outcome: 'ready',
+ connection: connection({ connectionKey: 'grok-aaa', displayName: 'Team Grok' }),
+ })
+ )
+
+ const dialog = await openGrokDialog()
+ await within(dialog).findByLabelText('grok-4.6')
+ fireEvent.click(within(dialog).getByRole('button', { name: 'Sync catalog' }))
+
+ // Liveness witness: the POST really went out and the re-read really ran and
+ // really rejected, so the copy below is the rule and not an unreached path.
+ await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1))
+ await waitFor(() => expect(listGrokConnectionModels).toHaveBeenCalledTimes(2))
+ expect(await screen.findByText('Catalog synced')).toBeInTheDocument()
+ expect(screen.queryByText(/Catalog sync failed/)).not.toBeInTheDocument()
+ expect(await screen.findByText(/could not be refreshed/)).toBeInTheDocument()
+ })
+
it('reports a non-ready outcome the endpoint returns with 200 instead of claiming success', async () => {
fetchMock.mockResolvedValueOnce(
makeResponse(200, {
diff --git a/control-ui/components/__tests__/LlmProviderConfig.test.tsx b/control-ui/components/__tests__/LlmProviderConfig.test.tsx
index 5f65db232..545947ab1 100644
--- a/control-ui/components/__tests__/LlmProviderConfig.test.tsx
+++ b/control-ui/components/__tests__/LlmProviderConfig.test.tsx
@@ -193,13 +193,16 @@ describe('LlmProviderConfig (spec Topic 1b — domain projection + usable gate)'
expect(screen.getByLabelText('Model', { selector: '#llm-primary-model' })).toBeInTheDocument()
})
- it('offers one OpenAI provider and does not invent a ChatGPT radio', () => {
+ // This picker chooses the CREDENTIAL PATH for an agent, so the API key and
+ // the subscription are peers and both appear. The /llm-models catalog is the
+ // surface that shows one row per family, and it collapses its own rows.
+ // Neither is a radio group: the credential path is a provider option, not a
+ // second control beside it.
+ it('offers the OpenAI API key and the OpenAI subscription as peer options', () => {
render(<Harness />)
fireEvent.click(screen.getByLabelText('Provider', { selector: '#llm-primary-provider' }))
expect(screen.getByRole('option', { name: /^OpenAI$/i })).toBeInTheDocument()
- expect(
- screen.queryByRole('option', { name: /OpenAI Codex Subscription/i })
- ).not.toBeInTheDocument()
+ expect(screen.getByRole('option', { name: /OpenAI Codex Subscription/i })).toBeInTheDocument()
expect(screen.queryByRole('radio', { name: /ChatGPT subscription/i })).not.toBeInTheDocument()
expect(screen.queryByRole('radio', { name: /API key/i })).not.toBeInTheDocument()
})
diff --git a/control-ui/lib/__tests__/llm.test.ts b/control-ui/lib/__tests__/llm.test.ts
index 0f3a7d358..c75f12deb 100644
--- a/control-ui/lib/__tests__/llm.test.ts
+++ b/control-ui/lib/__tests__/llm.test.ts
@@ -6,11 +6,9 @@ import {
LLM_DEFAULT_MODEL_BY_PROVIDER,
LLM_SECRET_EDITOR_GROUPS,
type LlmModelCatalogEntry,
- OPERATOR_PROVIDER_OPTIONS,
brokerBackedRecipeAuthoringError,
budgetUnitAllowedForProviders,
catalogGroupKey,
- familyCredentialSources,
getAllModelOptions,
getLlmGroupCompleteness,
getModelOptions,
@@ -19,6 +17,7 @@ import {
inferProviderFromModels,
isOpenAiFamily,
llmChainRequiresSecret,
+ operatorProviderOptions,
providerRequiresLlmSecret,
resolveDefaultModel,
validateLlmSecretData,
@@ -70,11 +69,25 @@ describe('getModelOptions', () => {
})
describe('OpenAI family presentation', () => {
- it('does not offer Codex as a second provider in operator pickers', () => {
- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'codex-subscription')).toBe(
- false
- )
- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'openai')).toBe(true)
+ // The agent-side credential picker offers a credential PATH, so an API key and
+ // a subscription are peers there and both families are treated alike. One row
+ // per family is the rule of the /llm-models catalog, which collapses its rows
+ // with `collapseFamilyRows` and never calls this function.
+ it('offers an API key and a subscription at the same level, in both families', () => {
+ const options = operatorProviderOptions({ grokEnabled: true })
+ expect(options.some(option => option.value === 'openai')).toBe(true)
+ expect(options.some(option => option.value === 'codex-subscription')).toBe(true)
+ expect(options.some(option => option.value === 'xai')).toBe(true)
+ expect(options.some(option => option.value === 'grok-subscription')).toBe(true)
+ })
+
+ // The Grok filter is a deployment capability gate, not a statement about
+ // families: Codex has no such signal in control-ui and is never gated.
+ it('hides only Grok, and only until the deployment gate is proven on', () => {
+ const gated = operatorProviderOptions()
+ expect(gated.some(option => option.value === 'grok-subscription')).toBe(false)
+ expect(gated.some(option => option.value === 'codex-subscription')).toBe(true)
+ expect(gated.some(option => option.value === 'xai')).toBe(true)
})
it('labels the subscription runtime id as OpenAI for operators', () => {
@@ -95,9 +108,6 @@ describe('OpenAI family presentation', () => {
expect(catalogGroupKey('xai')).toBe('xai')
expect(isOpenAiFamily('grok-subscription')).toBe(false)
expect(getProviderLabel('grok-subscription')).not.toBe('OpenAI')
- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'grok-subscription')).toBe(
- true
- )
})
it('keeps the broker label distinct from its family label', () => {
@@ -113,42 +123,6 @@ describe('OpenAI family presentation', () => {
// not be folded into anything.
expect(catalogGroupKey('not-a-provider')).toBe('not-a-provider')
})
-
- it('marks a model as API key, subscription, or both, in any family', () => {
- const catalog: LlmModelCatalogEntry[] = [
- { provider: 'openai', model: 'gpt-5.1', enabled: true },
- { provider: 'codex-subscription', model: 'gpt-5.1', enabled: true },
- { provider: 'codex-subscription', model: 'gpt-5.3-codex', enabled: true },
- { provider: 'xai', model: 'grok-4.6', enabled: true },
- { provider: 'grok-subscription', model: 'grok-4.6', enabled: true },
- { provider: 'grok-subscription', model: 'grok-4.6-fast', enabled: true },
- ]
- expect(familyCredentialSources(catalog, 'openai', 'gpt-5.1')).toEqual({
- apiKey: true,
- subscription: true,
- })
- expect(familyCredentialSources(catalog, 'openai', 'gpt-5.3-codex')).toEqual({
- apiKey: false,
- subscription: true,
- })
- expect(familyCredentialSources(catalog, 'xai', 'grok-4.6')).toEqual({
- apiKey: true,
- subscription: true,
- })
- expect(familyCredentialSources(catalog, 'xai', 'grok-4.6-fast')).toEqual({
- apiKey: false,
- subscription: true,
- })
- // A family does not see another family's rows.
- expect(familyCredentialSources(catalog, 'openai', 'grok-4.6')).toEqual({
- apiKey: false,
- subscription: false,
- })
- expect(familyCredentialSources(catalog, 'not-a-provider', 'grok-4.6')).toEqual({
- apiKey: false,
- subscription: false,
- })
- })
})
describe('getAllModelOptions', () => {
diff --git a/control-ui/lib/grokSubscription.ts b/control-ui/lib/grokSubscription.ts
index 5b82d68db..95f4c6e17 100644
--- a/control-ui/lib/grokSubscription.ts
+++ b/control-ui/lib/grokSubscription.ts
@@ -23,7 +23,14 @@ export type GrokCatalogStatus = CodexCatalogStatus
export type GrokConnectionStatus = CodexConnectionStatus
export type GrokOAuthIntent = CodexOAuthIntent
export type GrokDevicePollView = CodexDevicePollView
-export type GrokCatalogSyncView = CodexCatalogSyncView
+/**
+ * Narrower than the Codex view on purpose. The Grok `catalog/sync` endpoint
+ * answers `{ outcome, connection }` and sends no `added`/`refreshed`/`staled`
+ * counters, so `sanitizeCodexCatalogSync` fills them with zeros that mean "the
+ * endpoint said nothing", not "nothing changed". Promising the fields would
+ * invite a caller to render those zeros as a result.
+ */
+export type GrokCatalogSyncView = Omit<CodexCatalogSyncView, 'added' | 'refreshed' | 'staled'>
export type GrokDeviceStartView = {
userCode: string
diff --git a/control-ui/lib/llm.ts b/control-ui/lib/llm.ts
index 7e45ec589..78037b3a6 100644
--- a/control-ui/lib/llm.ts
+++ b/control-ui/lib/llm.ts
@@ -5,7 +5,6 @@ import {
PROVIDER_DISPLAY_LABELS,
PROVIDER_IDS,
PROVIDER_NON_SECRET_ENV,
- familyProviderIds,
isLlmProviderId,
providerFamily,
} from '@clerum/llm-providers'
@@ -51,7 +50,6 @@ export const LLM_PROVIDER_OPTIONS: Array<{ value: LlmProvider; label: string }>
id => ({ value: id, label: PROVIDER_DISPLAY_LABELS[id] })
)
-/** Runtime broker id. Never offer this as a second Provider row in operator pickers. */
export const OPENAI_SUBSCRIPTION_PROVIDER = 'codex-subscription' as const
export const GROK_SUBSCRIPTION_PROVIDER = 'grok-subscription' as const
@@ -69,15 +67,25 @@ export function isOauthBrokerProvider(provider: string | undefined | null): bool
)
}
-/** Provider dropdown: one OpenAI entry. Runtime still persists `codex-subscription`. */
-export const OPERATOR_PROVIDER_OPTIONS: Array<{ value: LlmProvider; label: string }> =
- LLM_PROVIDER_OPTIONS.filter(option => option.value !== OPENAI_SUBSCRIPTION_PROVIDER)
-
-/** Hide Grok until Control API proves the flag is on. */
+/**
+ * The agent-side credential picker (`LlmPolicyEditor`, `LlmProviderConfig`):
+ * every provider id, API keys and subscription brokers at the same level,
+ * because that is the choice being made — which credential path this agent
+ * uses. Both families are treated identically: `openai` beside
+ * `codex-subscription`, `xai` beside `grok-subscription`.
+ *
+ * One row per family is the rule of a DIFFERENT surface — the `/llm-models`
+ * catalog, where `collapseFamilyRows` groups the models of a family under one
+ * provider row with an `API key · Subscription` badge. That surface does not
+ * call this function.
+ *
+ * The only filter left is the Grok deployment gate: a capability the Control
+ * API has to confirm, not a statement about families.
+ */
export function operatorProviderOptions(opts?: {
grokEnabled?: boolean
}): Array<{ value: LlmProvider; label: string }> {
- return OPERATOR_PROVIDER_OPTIONS.filter(
+ return LLM_PROVIDER_OPTIONS.filter(
option => option.value !== GROK_SUBSCRIPTION_PROVIDER || opts?.grokEnabled === true
)
}
@@ -113,33 +121,6 @@ export function catalogGroupKey(provider: string): string {
return isLlmProviderId(provider) ? providerFamily(provider) : provider
}
-export type FamilyCredentialSource = 'api-key' | 'subscription'
-
-/**
- * Which credential paths serve a model inside one family. Derived from each
- * provider's auth mode rather than named ids, so a family gains a broker without
- * this function changing. An unknown family has no members, so neither path
- * serves it.
- */
-export function familyCredentialSources(
- catalog: LlmModelCatalogEntry[],
- family: string,
- model: string
-): { apiKey: boolean; subscription: boolean } {
- const members = familyProviderIds(family)
- const served = (mode: (typeof PROVIDER_AUTH_MODE)[LlmProviderId]): boolean =>
- catalog.some(
- row =>
- row.model === model &&
- row.enabled &&
- !row.stale &&
- isLlmProviderId(row.provider) &&
- members.includes(row.provider) &&
- PROVIDER_AUTH_MODE[row.provider] === mode
- )
- return { apiKey: served('static-credentials'), subscription: served('oauth-broker') }
-}
-
// The list of usable models per provider is no longer a static catalog: it is
// the operator-declared allowlist served by control-api (`/admin/llm-models`,
// spec §3-R3). Fetch it with `useLlmAllowedModels()` and pass the rows to the
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index 8936b7013..7118ebe17 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.606",
+ "version": "0.1.607",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.606",
+ "version": "0.1.607",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 93db41c74..d4848f328 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.606",
+ "version": "0.1.607",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/packages/llm-providers/index.d.ts b/packages/llm-providers/index.d.ts
index ccb5508e4..99c20f0db 100644
--- a/packages/llm-providers/index.d.ts
+++ b/packages/llm-providers/index.d.ts
@@ -150,8 +150,12 @@ export declare function buildProviderMaps(
PROVIDER_AUTH_MODE: Readonly<Record<string, ProviderAuthMode>>
PROVIDER_MODEL_CATALOG_MODE: Readonly<Record<string, ProviderModelCatalogMode>>
}
-/** Every provider id → the provider id that owns its family. Total, frozen. */
-export declare const PROVIDER_FAMILY: Record<LlmProviderId, LlmProviderId>
+/**
+ * Every provider id → the provider id that owns its family. Total, frozen.
+ * `Readonly` because the value really is `Object.freeze`d: a plain `Record`
+ * type-checks a write that then silently no-ops at runtime.
+ */
+export declare const PROVIDER_FAMILY: Readonly<Record<LlmProviderId, LlmProviderId>>
/** The family of a known provider. Throws on an unrecognised id. */
export declare function providerFamily(id: LlmProviderId): LlmProviderId
/**
diff --git a/scripts/e2e/e2e-codex-subscription-playwright.sh b/scripts/e2e/e2e-codex-subscription-playwright.sh
index dc693e7e8..5ec33b559 100755
--- a/scripts/e2e/e2e-codex-subscription-playwright.sh
+++ b/scripts/e2e/e2e-codex-subscription-playwright.sh
@@ -147,6 +147,29 @@ fi
wait_http control-ui "${CONTROL_UI_URL%/}/" 90
wait_http control-api "${CONTROL_API_URL%/}/health" 60
+# Fail-closed deployment preconditions for the catalog re-sync guardian.
+#
+# control-api collapses "proxy admin URL unset", "proxy unreachable" and "the
+# vendor answered badly" into one identical response — 503, catalog_sync_failed,
+# outcome `unavailable` — so no assertion in the spec can separate them. A lane
+# missing either precondition would render an error banner and
+# codex-subscription-catalog-resync.spec.ts would pass its 503 branch while no
+# operator could ever sync a catalog. Checked here, before a browser starts,
+# where the diagnosis is unambiguous and costs two API reads.
+log "Verifying Codex catalog sync deployment preconditions"
+proxy_admin_url="$(kubectl --context="${PROFILE}" -n control-plane get configmap control-api-config \
+ -o jsonpath='{.data.CODEX_LLM_PROXY_ADMIN_URL}')" \
+ || die "cannot read configmap/control-api-config in namespace control-plane"
+[[ -n "${proxy_admin_url}" ]] \
+ || die "CODEX_LLM_PROXY_ADMIN_URL is empty in configmap/control-api-config; control-api would answer every catalog sync with 503 unavailable without ever reaching the proxy"
+
+proxy_ready="$(kubectl --context="${PROFILE}" -n control-plane get deployment codex-llm-proxy \
+ -o jsonpath='{.status.readyReplicas}')" \
+ || die "cannot read deployment/codex-llm-proxy in namespace control-plane"
+[[ "${proxy_ready:-0}" -ge 1 ]] \
+ || die "deployment/codex-llm-proxy has no ready replica (readyReplicas='${proxy_ready:-0}'); control-api would answer every catalog sync with 503 unavailable"
+log "Preconditions OK (proxy admin URL set, codex-llm-proxy readyReplicas=${proxy_ready})"
+
log "Running Control UI Playwright guardians (CONTROL_UI_URL=${CONTROL_UI_URL})"
(
cd "${PLAYWRIGHT_DIR}"
diff --git a/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts
index 2f373740a..f7eb6c939 100644
--- a/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts
+++ b/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts
@@ -13,9 +13,17 @@
* for the rows and the union allowlist.
*
* What this lane uniquely proves: the affordance exists on a connected grant
- * and nowhere else, the click really issues the write, and the window settles
- * on the answer the API gave instead of leaving the operator with a spinner
- * and no verdict.
+ * and nowhere else, the click really issues the write, the write is RECORDED,
+ * and the window settles on the answer the API gave instead of leaving the
+ * operator with a spinner and no verdict.
+ *
+ * Its honest limit: control-api collapses "proxy admin URL unset", "proxy
+ * unreachable" and "chatgpt.com answered badly" into the same
+ * `503 / catalog_sync_failed / unavailable`, so no assertion at this lane's
+ * HTTP surface can separate a vendor outage from a broken deployment. Those two
+ * are deployment preconditions and are checked fail-closed by the lane runner
+ * (`scripts/e2e/e2e-codex-subscription-playwright.sh`) BEFORE Playwright
+ * starts, which is why the 503 branch below can stay green on `unavailable`.
*
* Auth: `/` + loginControlUiVisible. No terminal `goto`. No test.skip.
*/
@@ -27,11 +35,29 @@ import { ControlUiShell, SecretsLlmSubscriptionsPage } from '../pages/codex-subs
const SYNC_PATH =
/\/api\/v1\/admin\/llm\/providers\/codex-subscription\/connections\/[^/]+\/catalog\/sync$/
-// The closed set the route answers with (control-api/src/routes/admin/
-// codexSubscription.ts:734-770). A status outside it means the contract moved
-// and the mapping in CodexSubscriptionHub is stale, which is worth failing on
-// even though the operator-visible outcome would still be "an error banner".
-const DOCUMENTED_SYNC_STATUS = [200, 401, 403, 404, 409, 503]
+// The closed set of outcomes a CORRECTLY PROVISIONED lane can answer with
+// (control-api/src/routes/admin/codexSubscription.ts). A status outside it
+// means either the contract moved or this lane's own environment is broken,
+// and both must be red.
+//
+// What is deliberately absent, and why an error banner alone is not enough to
+// pass:
+// - 401/403: the lane's admin session was refused. Not an outcome of this
+// feature at all.
+// - 404: `disabled` is unreachable because the list route answers 404 too, so
+// `listCodexConnections()` would have thrown before the click; `no_grant`
+// means the connection row exists without its secrets, i.e. lane state
+// corruption.
+// - 503 with any outcome other than `unavailable`: `auth-rejected` (dead grant
+// token or a broken proxy permit), `never_synced` (row disconnected under
+// us, DB error) and `configmap_write_failed` are all this lane's own
+// breakage wearing a product-shaped banner.
+const DOCUMENTED_SYNC_STATUS = [200, 409, 503]
+
+// One spelling, shared by the affordance assertion in R1 and the ABSENCE
+// assertion in R2. Two literals would let a rename leave R2 asserting that a
+// button nobody renders is not rendered — green, and proving nothing.
+const SYNC_BUTTON = 'Sync catalog'
async function loginFromHome(page: Page) {
await page.goto('/')
@@ -77,7 +103,7 @@ test.describe('Codex subscription catalog re-sync', () => {
await modelsGet
const dialog = page.getByRole('dialog')
- const syncButton = dialog.getByRole('button', { name: 'Sync catalog' })
+ const syncButton = dialog.getByRole('button', { name: SYNC_BUTTON })
await expect(syncButton).toBeVisible()
await expect(syncButton).toBeEnabled()
@@ -99,25 +125,48 @@ test.describe('Codex subscription catalog re-sync', () => {
const errorBanner = dialog.locator('.cu-banner--error')
if (syncRes.status() === 200) {
+ // `ok: true` carries `catalogStatus: 'ready'` and nothing else
+ // (codexSubscriptionOAuth.ts CodexCatalogSyncResult), so a 200 with any
+ // other outcome is a contract move, not a degraded sync. The old
+ // non-ready arm here was unreachable.
const body = (await syncRes.json()) as { outcome?: string }
+ expect(body.outcome).toBe('ready')
// A recorded outcome always advances catalog_revision
// (codexSubscriptionConnection.ts:692), so this is the business signal
// that the write landed rather than the UI merely redrawing.
expect(await catalogRevisionOf(grantKey)).toBeGreaterThan(revisionBefore)
- if (body.outcome === 'ready') {
- await expect(successToast).toBeVisible()
- await expect(errorBanner).toHaveCount(0)
- } else {
- await expect(
- page.getByRole('status').filter({ hasText: 'Catalog sync failed' })
- ).toBeVisible()
- }
- } else {
- // The operator must be told. An empty banner is the failure this guards:
- // the modal would otherwise look identical to a successful sync.
+ await expect(successToast).toBeVisible()
+ await expect(errorBanner).toHaveCount(0)
+ } else if (syncRes.status() === 409) {
+ // A concurrent writer won the revision or refresh-lock fence — the
+ // reconciliation cron runs the very same runCodexCatalogSync on Codex
+ // grants, so it can race this click. The click still issued the write;
+ // this request recorded nothing, so the revision is NOT asserted.
+ const body = (await syncRes.json()) as { error?: string }
+ expect(['stale_revision', 'refresh_in_flight']).toContain(body.error)
await expect(errorBanner).toBeVisible()
await expect(errorBanner).not.toHaveText('')
await expect(successToast).toHaveCount(0)
+ } else {
+ // 503 is accepted for exactly one outcome: the vendor answered badly.
+ // That is the only non-200 this lane cannot control, because the catalog
+ // origin is frozen to chatgpt.com by codex-llm-proxy/src/originPolicy.ts
+ // and this lane deploys no fake upstream.
+ expect(syncRes.status()).toBe(503)
+ const body = (await syncRes.json()) as { error?: string; outcome?: string }
+ expect(body.error).toBe('catalog_sync_failed')
+ expect(body.outcome).toBe('unavailable')
+ // The outcome was RECORDED, so the revision advanced: the sync ran end to
+ // end and only the upstream failed. A transport that never ran — a dead
+ // grant token, a disconnected row, a failed ConfigMap write — cannot
+ // produce this, which is what keeps the branch from passing on breakage.
+ expect(await catalogRevisionOf(grantKey)).toBeGreaterThan(revisionBefore)
+ // The operator must be told, and told WHAT: an empty or generic banner is
+ // the failure this guards, because the modal would otherwise look
+ // identical to a successful sync.
+ await expect(errorBanner).toBeVisible()
+ await expect(errorBanner).toContainText('unavailable')
+ await expect(successToast).toHaveCount(0)
}
// Whatever the outcome, the control comes back: a stuck busy state would
@@ -145,7 +194,7 @@ test.describe('Codex subscription catalog re-sync', () => {
expect(created, 'the grant under test must be listed').toBeTruthy()
expect(created!.status).not.toBe('connected')
- await expect(dialog.getByRole('button', { name: 'Sync catalog' })).toHaveCount(0)
+ await expect(dialog.getByRole('button', { name: SYNC_BUTTON })).toHaveCount(0)
await hub.closeConnectModal()
})
diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.json
index 7acdea7b6..d921d1cec 100644
--- a/tests/e2e/playwright/package-lock.json
+++ b/tests/e2e/playwright/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-playwright-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"devDependencies": {
"@playwright/test": "^1.50.0",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.json
index 8260c6483..d7b65879c 100644
--- a/tests/e2e/playwright/package.json
+++ b/tests/e2e/playwright/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"private": true,
"description": "Playwright E2E tests for Control UI and Desktop App",
"scripts": {