Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(llm): publish the runtime allowlist on every reconciliation tick
Alfredo Lopezf603cc927397
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 6ae781fe2..343c05cc9 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.596","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.596","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 5b35eb47d..15acddefd 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.596","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/main.ts b/control-api/src/main.tsindex 4571932f1..d12a0a357 100644--- a/control-api/src/main.ts+++ b/control-api/src/main.ts@@ -126,8 +126,11 @@ async function main(): Promise<void> {// afterwards stays invisible to the subscription while the API-key provider of// the SAME vendor picks it up from the discovery sync above. The tick re-runs// the identical per-connection sync the Hub's manual action drives, and needs- // the gateway for the same reason: it publishes the allowlist ConfigMap when a- // connection records an outcome. Per-broker gates still apply inside the tick.+ // the gateway for the same reason: it publishes the allowlist ConfigMap on+ // every tick — unconditionally, like `reconcileAllowedModelsConfigMapOnBoot`+ // above, so a publish that threw converges on the next tick instead of+ // stranding a stale runtime snapshot. Per-broker gates still apply inside the+ // tick.if (config.subscriptionCatalogSyncCronEnabled) {startSubscriptionCatalogSyncCron({ sync: () => reconcileSubscriptionCatalogsFromEnv(gateway.llmAllowedModelsConfigMap()) },diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.tsindex 9992f09c1..4ba0d6058 100644--- a/control-api/src/routes/admin/codexSubscription.ts+++ b/control-api/src/routes/admin/codexSubscription.ts@@ -77,7 +77,10 @@ import {runGrokCatalogSync,startGrokDeviceConnect,} from '../../services/grokSubscriptionOAuth.js'-import { publishAllowedModelsConfigMapAfterGrantChange } from '../../services/llmAllowedModelsConfigMap.js'+import {+ publishAllowedModelsConfigMapAfterGrantChange,+ syncOutcomeChangedTheRow,+} from '../../services/llmAllowedModelsConfigMap.js'import { K8sConflictError } from '../../services/resourceService.js'import {adminCodexReadRateLimits,@@ -701,6 +704,17 @@ export function createAdminCodexSubscriptionRouter(})return}+ // The publish comes BEFORE the error responses, because a failed sync+ // can still have written the connection row — a rejected refresh token+ // moves it to `reauth_required` and only then throws, which surfaces+ // here as `never_synced` plus `persisted`. mcp-host and HCC never read+ // Postgres, so an unpublished write leaves the runtime serving a grant+ // the control plane already knows is broken, whatever status code this+ // handler picks afterwards. `syncOutcomeChangedTheRow` is the same rule+ // the reconciliation cron applies.+ if (syncOutcomeChangedTheRow(synced)) {+ if (await publishRuntimeAllowlistOrFail(res)) return+ }if (synced.reason === 'no_grant' || synced.reason === 'disabled') {res.status(404).json({ error: synced.reason })return@@ -716,14 +730,6 @@ export function createAdminCodexSubscriptionRouter()return}- // A catalogStatus other than `never_synced` means the sync reached xAI- // and the connection row now carries that outcome, so the runtime- // snapshot is stale until it is republished — mcp-host and HCC never- // read Postgres. `never_synced` persisted nothing and needs no write.- // The Codex branch below publishes before its own 503 for this reason.- if (synced.catalogStatus !== 'never_synced') {- if (await publishRuntimeAllowlistOrFail(res)) return- }res.status(503).json({ error: 'catalog_sync_failed', outcome: synced.catalogStatus })return}@@ -760,10 +766,21 @@ export function createAdminCodexSubscriptionRouter(sendOAuthError(res, new CodexSubscriptionOAuthError(synced.reason, synced.reason))return}- if (await publishRuntimeAllowlistOrFail(res)) return+ // Same rule as the Grok branch above. No Codex error path writes the+ // connection row — its only status writes are `cancelled` and `revoked`,+ // both explicit operator actions — so in practice this publishes exactly+ // when `catalogStatus` recorded something, and skips the write when the+ // sync recorded nothing.+ if (syncOutcomeChangedTheRow(synced)) {+ if (await publishRuntimeAllowlistOrFail(res)) return+ }res.status(503).json({error: 'catalog_sync_failed',- outcome: synced.catalogStatus === 'never_synced' ? 'unavailable' : synced.catalogStatus,+ // Report what the sync reported. Rewriting `never_synced` as+ // `unavailable` maps "nothing was persisted" onto a status that means+ // "we reached the vendor and it was down", and the Hub renders this+ // string to the operator verbatim.+ outcome: synced.catalogStatus,added: synced.added ?? 0,refreshed: synced.refreshed ?? 0,staled: synced.staled ?? 0,diff --git a/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts b/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.tsindex 83712cdd1..8285f310f 100644--- a/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts+++ b/control-api/src/services/__tests__/subscriptionCatalogSyncCron.test.ts@@ -6,8 +6,9 @@* piling up redundant upstream calls. This suite pins the hazards that make the* difference between a reconciler and an abuse vector: the lock lifecycle, the* skip rules (a disabled broker, a grant that is not `connected`), failure- * isolation per connection, and the single ConfigMap publish whose rule is- * "any RECORDED outcome", not "a ready outcome".+ * isolation per connection, and the single ConfigMap publish — which runs on+ * every tick, including one that recorded nothing, because that is the tick+ * after a publish that threw and the rows it changed are skipped by status.** Deps are injected: no database, no cluster, no upstream. The production* wiring is asserted where it belongs — the route suite for the endpoint and@@ -38,6 +39,19 @@ const STALE: SubscriptionCatalogSyncOutcome = {catalogStatus: 'never_synced',reason: 'stale_revision',}+/**+ * The write landed and the sync still reports `never_synced`.+ * `markGrokRefreshSubjectMismatch` sets `status = 'reauth_required'` and THEN+ * throws, so the catalog was never synced while the CONNECTION row changed.+ * `llmAllowedModelsConfigMap` maps that status into the ConfigMap, so the+ * publish is owed — `never_synced` alone cannot decide it.+ */+const PERSISTED_REAUTH: SubscriptionCatalogSyncOutcome = {+ ok: false,+ catalogStatus: 'never_synced',+ reason: 'reauth_required',+ persisted: true,+}/*** A fake pool sharing one advisory-lock state across every client it hands out@@ -111,6 +125,7 @@ function makeMaterializer(): AllowedModelsConfigMapMaterializer & {afterEach(() => {stopSubscriptionCatalogSyncCron()vi.useRealTimers()+ vi.unstubAllGlobals()})describe('subscription catalog sync tick — advisory lock', () => {@@ -319,7 +334,7 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {expect(materializer.materialize).toHaveBeenCalledTimes(1)})- it('does not publish when nothing recorded an outcome', async () => {+ it('publishes although nothing recorded an outcome, so a failed publish converges', async () => {const codex = makePort('codex-subscription',[@@ -334,12 +349,74 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {// Liveness witness: the loop ran and reached the broker — one row was// skipped by status and the other really was synced and raced — so the- // absent publish is the rule under test, not an unreached path.+ // publish under test happened after a tick that wrote nothing, which is+ // the whole point. This shape is the tick AFTER a publish that threw: the+ // rows it changed are `reauth_required` by then and get skipped here, so+ // gating the publish on "something was recorded" would strand the stale+ // ConfigMap forever.expect(codex.listConnections).toHaveBeenCalledTimes(1)expect(codex.syncCatalog).toHaveBeenCalledWith('raced')expect(result).toMatchObject({ skipped: 1, raced: 1 })- expect(materializer.materialize).not.toHaveBeenCalled()- expect(result.published).toBe('skipped')+ expect(materializer.materialize).toHaveBeenCalledTimes(1)+ expect(result.published).toBe('published')+ })++ it('reports skipped when no materializer is wired, without calling one', async () => {+ const codex = makePort(+ 'codex-subscription',+ [{ connectionKey: 'one', status: 'connected' }],+ async () => READY+ )++ const result = await reconcileSubscriptionCatalogs({+ brokers: [codex],+ materializer: undefined,+ })++ // Liveness witness for the negative: the sync really ran, so `skipped`+ // here is the absent writer and not an abandoned tick.+ expect(codex.syncCatalog).toHaveBeenCalledWith('one')+ expect(result).toMatchObject({ synced: 1, published: 'skipped' })+ })++ it('publishes when the sync persisted a status although it reports never_synced', async () => {+ const grok = makePort(+ 'grok-subscription',+ [{ connectionKey: 'rejected', status: 'connected' }],+ async () => PERSISTED_REAUTH+ )+ const materializer = makeMaterializer()++ const result = await reconcileSubscriptionCatalogs({ brokers: [grok], materializer })++ // Liveness witness: the loop reached the broker and ran the sync, so the+ // publish below is the rule under test rather than an unvisited path.+ expect(grok.syncCatalog).toHaveBeenCalledWith('rejected')+ // The row carries `reauth_required` now. Counting this as `failed` and+ // skipping the publish leaves mcp-host and HCC serving `connected` with the+ // full model list until an unrelated grant mutation republishes.+ expect(result).toMatchObject({ degraded: 1, failed: 0, published: 'published' })+ expect(materializer.materialize).toHaveBeenCalledTimes(1)+ })++ it('contains a broker whose connection listing fails, so the other broker still runs', async () => {+ const codex = makePort('codex-subscription', [], async () => READY)+ codex.listConnections.mockRejectedValueOnce(new Error('connection listing failed'))+ const grok = makePort(+ 'grok-subscription',+ [{ connectionKey: 'team-grok', status: 'connected' }],+ async () => READY+ )+ const materializer = makeMaterializer()++ const result = await reconcileSubscriptionCatalogs({ brokers: [codex, grok], materializer })++ // `listConnections` runs OUTSIDE the per-connection try, so without its own+ // guard this rejection escapes the broker loop and the whole tick is lost —+ // including every Grok connection, which has nothing to do with it.+ expect(codex.listConnections).toHaveBeenCalledTimes(1)+ expect(result).toMatchObject({ failed: 1, synced: 1, published: 'published' })+ expect(grok.syncCatalog).toHaveBeenCalledWith('team-grok')})it('records the tick as failed when the publish itself fails, without throwing', async () => {@@ -365,6 +442,25 @@ describe('subscription catalog reconciliation — ConfigMap publish', () => {describe('subscription catalog cron scheduling', () => {it('runs a first jittered tick and then one per interval, with unref()d handles', async () => {vi.useFakeTimers()+ // The title claims unref(), so the test has to check it. An un-unref'd+ // handle keeps the event loop alive and control-api stops exiting on+ // SIGTERM — a failure that never shows up in a test that only counts ticks.+ const unrefed: string[] = []+ const realSetTimeout = globalThis.setTimeout+ const realSetInterval = globalThis.setInterval+ const recordUnref = <T extends { unref: () => T }>(handle: T, label: string): T => {+ const original = handle.unref.bind(handle)+ handle.unref = () => {+ unrefed.push(label)+ return original()+ }+ return handle+ }+ vi.stubGlobal('setTimeout', ((fn: () => void, ms?: number) =>+ recordUnref(realSetTimeout(fn, ms), 'firstRun')) as unknown as typeof setTimeout)+ vi.stubGlobal('setInterval', ((fn: () => void, ms?: number) =>+ recordUnref(realSetInterval(fn, ms), 'interval')) as unknown as typeof setInterval)+const shared = { held: false }const calls: string[] = []const connector = makeLockPool(shared, calls)@@ -381,6 +477,9 @@ describe('subscription catalog cron scheduling', () => {await vi.advanceTimersByTimeAsync(5_000)expect(sync).toHaveBeenCalledTimes(1)+ // The first-run handle is unref'd before the timer fires; the interval+ // handle only exists after it.+ expect(unrefed).toEqual(['firstRun', 'interval'])await vi.advanceTimersByTimeAsync(60_000)expect(sync).toHaveBeenCalledTimes(2)diff --git a/control-api/src/services/grokSubscriptionOAuth.ts b/control-api/src/services/grokSubscriptionOAuth.tsindex 5685bd9c9..4a0387cac 100644--- a/control-api/src/services/grokSubscriptionOAuth.ts+++ b/control-api/src/services/grokSubscriptionOAuth.ts@@ -118,10 +118,24 @@ export function isGrokOAuthErrorCode(value: string): value is GrokOAuthErrorCodeexport class GrokSubscriptionOAuthError extends Error {readonly code: GrokOAuthErrorCode- constructor(code: GrokOAuthErrorCode, message: string) {+ /**+ * True when this throw FOLLOWS a write to the connection row, so the caller+ * must treat the connection as changed even though the operation failed. A+ * rejected refresh token is the case that matters: the row is moved to+ * `reauth_required` and only then does the error propagate, so the runtime+ * snapshot is stale until it is republished.+ */+ readonly persistedConnectionStatus: boolean++ constructor(+ code: GrokOAuthErrorCode,+ message: string,+ opts: { persistedConnectionStatus?: boolean } = {}+ ) {super(message)this.name = 'GrokSubscriptionOAuthError'this.code = code+ this.persistedConnectionStatus = opts.persistedConnectionStatus === true}}@@ -205,6 +219,12 @@ export type GrokCatalogSyncResult =ok: falsecatalogStatus: GrokCatalogOutcome | 'never_synced'reason?: GrokOAuthErrorCode | 'catalog_sync_failed' | 'stale_revision' | 'no_grant'+ /**+ * The catalog was not synced but the CONNECTION row was written anyway.+ * `catalogStatus` describes the catalog, so it cannot answer "did the row+ * change?" — only this field can.+ */+ persisted?: boolean}export async function getGrokSubscriptionConnection(@@ -449,7 +469,9 @@ export async function runGrokCatalogSync()const reason: GrokOAuthErrorCode | 'catalog_sync_failed' =err instanceof GrokSubscriptionOAuthError ? err.code : 'catalog_sync_failed'- return { ok: false, catalogStatus: 'never_synced', reason }+ const persisted =+ err instanceof GrokSubscriptionOAuthError && err.persistedConnectionStatus === true+ return { ok: false, catalogStatus: 'never_synced', reason, persisted }}}@@ -687,7 +709,12 @@ async function exchangeRefreshToken()}await markGrokRefreshSubjectMismatch(deps.db, fence.connectionKey, fence.expectedRevision)- throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected')+ // The row is now `reauth_required`. The throw below must not read as+ // "nothing happened": the ConfigMap carries this status, so whoever+ // catches this owes a republish.+ throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {+ persistedConnectionStatus: true,+ })}throw new GrokSubscriptionOAuthError('provider_unavailable', 'refresh token exchange failed')}diff --git a/control-api/src/services/llmAllowedModelsConfigMap.ts b/control-api/src/services/llmAllowedModelsConfigMap.tsindex d818842ae..922dcd47c 100644--- a/control-api/src/services/llmAllowedModelsConfigMap.ts+++ b/control-api/src/services/llmAllowedModelsConfigMap.ts@@ -172,6 +172,30 @@ export interface AllowedModelsConfigMapMaterializer {** Missing writer is a no-op so unit tests without a K8s gateway stay local.*/+/**+ * Did this catalog sync change the connection row, and therefore owe a publish?+ *+ * The rule belongs here, beside the publish it gates, because it was previously+ * written inline in three places in three shapes — the cron, the Grok route and+ * the Codex route — and one of them was wrong.+ *+ * `catalogStatus` describes the CATALOG, so it cannot answer this on its own. A+ * rejected Grok refresh token writes `status = 'reauth_required'` to the row and+ * only then throws, leaving `catalogStatus: 'never_synced'` on a row that did+ * change; `mapCodexConnectionStatusForSnapshot` puts that status in the+ * ConfigMap, so a skipped publish leaves mcp-host and HCC serving a grant the+ * control plane already knows is broken. `persisted` is the services' explicit+ * answer for exactly that case.+ */+export function syncOutcomeChangedTheRow(outcome: {+ ok: boolean+ catalogStatus: string+ persisted?: boolean+}): boolean {+ if (outcome.ok) return true+ return outcome.persisted === true || outcome.catalogStatus !== 'never_synced'+}+export async function publishAllowedModelsConfigMapAfterGrantChange(writer: AllowedModelsConfigMapMaterializer | undefined): Promise<'published' | 'skipped'> {diff --git a/control-api/src/services/subscriptionCatalogSyncCron.ts b/control-api/src/services/subscriptionCatalogSyncCron.tsindex a821ebd92..2c62335e7 100644--- a/control-api/src/services/subscriptionCatalogSyncCron.ts+++ b/control-api/src/services/subscriptionCatalogSyncCron.ts@@ -48,7 +48,10 @@ import {} from './grokSubscriptionConnection.js'import { runGrokCatalogSync } from './grokSubscriptionOAuth.js'import type { AllowedModelsConfigMapMaterializer } from './llmAllowedModelsConfigMap.js'-import { publishAllowedModelsConfigMapAfterGrantChange } from './llmAllowedModelsConfigMap.js'+import {+ publishAllowedModelsConfigMapAfterGrantChange,+ syncOutcomeChangedTheRow,+} from './llmAllowedModelsConfigMap.js'const log = rootLogger.child({ service: 'subscription_catalog_sync_cron' })@@ -59,14 +62,19 @@ const CRON_LOCK_KEY_SQL = "hashtext('subscription-catalog-sync-cron-v1')"export type SubscriptionBroker = 'codex-subscription' | 'grok-subscription'/**- * What `runCodexCatalogSync` / `runGrokCatalogSync` return, reduced to the two- * fields this module decides on. `catalogStatus: 'never_synced'` is the- * services' own signal that NOTHING was persisted for the connection; any other- * status means the row now carries that outcome.+ * What `runCodexCatalogSync` / `runGrokCatalogSync` return, reduced to the+ * fields this module decides on.+ *+ * `catalogStatus` describes the CATALOG, so it cannot answer "did the connection+ * row change?" on its own: a rejected Grok refresh token writes+ * `status = 'reauth_required'` and only then throws, leaving the catalog at+ * `never_synced` on a row that did change. `persisted` is the services' explicit+ * answer for that case, and `syncOutcomeChangedTheRow` is the single place that+ * reads both.*/export type SubscriptionCatalogSyncOutcome =| { ok: true; catalogStatus: 'ready' }- | { ok: false; catalogStatus: string; reason?: string }+ | { ok: false; catalogStatus: string; reason?: string; persisted?: boolean }export type SubscriptionConnectionRow = { connectionKey: string; status: string }@@ -134,7 +142,9 @@ export function filterAddressableGrokConnections<T extends SubscriptionConnectio* others their sync. The publish, in contrast, is not isolated — a catalog that* changed in Postgres but not in the ConfigMap is a divergence the runtime hosts* would serve until the next mutation, so the rejection propagates and the tick- * reports `errored`.+ * reports `errored`. It runs on every tick regardless of what the loop recorded,+ * which is what makes the next tick repair a publish that threw; see the comment+ * at the publish itself.*/export async function reconcileSubscriptionCatalogs(deps: SubscriptionCatalogReconcileDeps@@ -153,7 +163,25 @@ export async function reconcileSubscriptionCatalogs()continue}- const rows = await port.listConnections()+ // Outside the per-connection `try` below, so without this guard a single+ // broker's listing failure would abandon the OTHER broker's connections+ // too — the tick would throw out of the loop entirely. One failed listing+ // is one failure, not a lost tick.+ let rows: SubscriptionConnectionRow[]+ try {+ rows = await port.listConnections()+ } catch (err) {+ failed += 1+ log.warn(+ {+ event: 'subscription_catalog_sync_listing_failed',+ broker: port.broker,+ err: err instanceof Error ? err.message : String(err),+ },+ `${port.broker} connection listing failed; continuing with the remaining brokers`+ )+ continue+ }for (const row of rows) {// Every narrower rule is wrong: `disconnected` makes the catalog sync// throw `not_connected`, `connecting` throws `no_grant` inside the@@ -167,7 +195,7 @@ export async function reconcileSubscriptionCatalogs(const outcome = await port.syncCatalog(row.connectionKey)if (outcome.ok) {synced += 1- } else if (outcome.catalogStatus !== 'never_synced') {+ } else if (syncOutcomeChangedTheRow(outcome)) {degraded += 1log.info({@@ -175,6 +203,10 @@ export async function reconcileSubscriptionCatalogs(broker: port.broker,connectionKey: row.connectionKey,catalogStatus: outcome.catalogStatus,+ // Present when the catalog stayed at `never_synced` and the+ // CONNECTION row is what changed — the reason is then the only+ // field that says what the row now carries.+ reason: outcome.reason,},`${port.broker} catalog recorded a non-ready outcome`)@@ -207,16 +239,25 @@ export async function reconcileSubscriptionCatalogs(}}- // Publish on ANY recorded outcome, ready or not. A tick that moves a grant to- // `reauth_required` persists that status, the ConfigMap carries it in its- // annotations, and the materializer's own contract requires the publish for- // "recorded non-ready outcomes". Publishing only on `ready` would leave the- // runtime hosts serving a grant the control plane already knows is broken.- const recorded = synced + degraded- const published =- recorded > 0- ? await publishAllowedModelsConfigMapAfterGrantChange(deps.materializer)- : 'skipped'+ // Publish on EVERY tick that ran, not only on one that wrote a row.+ //+ // `materialize()` rebuilds the ConfigMap from Postgres — the allowlist plus+ // both brokers' live connections and their readiness — so the write is+ // idempotent and publishing unconditionally costs one API call per interval.+ // That is what lets a publish that threw converge on the next tick with no+ // state carried across ticks or replicas: the advisory lock is taken per+ // tick, so a flag set by the replica that failed would be invisible to+ // whichever replica wins the next one.+ //+ // A "publish only when something was recorded" gate cannot do that job. The+ // rows a failed tick changed are `reauth_required` by the next tick, the+ // loop above skips them by status, nothing is recorded, and the stale+ // ConfigMap — where mcp-host and HCC still see a usable grant — survives+ // until an unrelated mutation or a control-api restart happens to republish.+ //+ // `llmAllowedModelsBootReconcile.ts` already writes unconditionally at boot+ // for the same anti-drift reason; this makes the cron agree with it.+ const published = await publishAllowedModelsConfigMapAfterGrantChange(deps.materializer)return { synced, degraded, raced, failed, skipped, published }}@@ -231,21 +272,24 @@ function cronDbClient() {** The laziness is load-bearing for Codex: `normalizeControlUiOrigin` throws on a* malformed, non-http or path-bearing `CONTROL_API_CONTROL_UI_BASE_URL`, and a- * throw at boot would take control-api down instead of failing one tick. The- * redirect URI is built only when Codex is enabled, so a bad value cannot cost- * Grok its reconciliation either. `runCodexCatalogSync` reaches the refresh path- * only — `exchangeRefreshToken` sends no `redirect_uri` — but a real value is- * built anyway rather than a placeholder that would be wrong the day the path- * changes.+ * throw at boot would take control-api down instead of failing one tick.+ *+ * The redirect URI is built inside `syncCatalog`, not here. Building it while+ * WIRING the ports would throw before the Grok port exists, so a Codex-only+ * misconfiguration would cost Grok its entire reconciliation — the failure this+ * function is meant to contain. Inside `syncCatalog` the throw is caught by the+ * per-connection `try` in `reconcileSubscriptionCatalogs` and counted as one+ * failed Codex connection, which is what it is.+ *+ * `runCodexCatalogSync` reaches the refresh path only — `exchangeRefreshToken`+ * sends no `redirect_uri` — but a real value is built anyway rather than a+ * placeholder that would be wrong the day the path changes.*/export function createSubscriptionBrokerPorts(): SubscriptionBrokerPort[] {const db = cronDbClient()const encryptionKey = deriveOAuthEncryptionKey(config.oauthEncryptionKey)const codexTransport = createCodexCatalogTransportFromEnv()- const codexRedirectUri = config.codexSubscriptionEnabled- ? buildCodexBrowserRedirectUri(resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, undefined))- : ''const codex: SubscriptionBrokerPort = {broker: 'codex-subscription',enabled: config.codexSubscriptionEnabled,@@ -257,7 +301,9 @@ export function createSubscriptionBrokerPorts(): SubscriptionBrokerPort[] {encryptionKey,fetchFn: fetch,clientId: config.codexOAuthClientId,- redirectUri: codexRedirectUri,+ redirectUri: buildCodexBrowserRedirectUri(+ resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, undefined)+ ),enabled: config.codexSubscriptionEnabled,connectionKey,},diff --git a/control-api/test/config.subscriptionCatalogSyncCron.test.ts b/control-api/test/config.subscriptionCatalogSyncCron.test.tsnew file mode 100644index 000000000..5c1a612b3--- /dev/null+++ b/control-api/test/config.subscriptionCatalogSyncCron.test.ts@@ -0,0 +1,151 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import { readFileSync } from 'node:fs'++/**+ * Deploy mirror for the subscription catalog reconciliation cron's config keys.+ *+ * The cron reads three keys. Two of them were added by this change and the+ * third is its older sibling, and nothing so far asserts that what+ * `config.ts` reads is what the cluster actually sets. A key renamed in code+ * and not in `configmaps.yaml` leaves the cron silently at its code default,+ * which for `SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED` means permanently off —+ * a reconciler that never runs and never says so.+ *+ * The assertions are NOT "every key equals the code default". Two of these+ * values are deliberately not the default, and pinning them to the default+ * would be wrong: `LLM_CATALOG_SYNC_CRON_ENABLED` is deliberately ON in the+ * cluster while its code default is off. Each key is pinned to the value the+ * deployment means, with the reason beside it.+ *+ * The pattern (`read` + a fail-loud single-match `extractOne`) follows+ * `config.pluginSdkRateLimits.test.ts`. A key that is absent from the manifest+ * makes `extractOne` throw rather than skipping the assertion.+ */++const CRON_KEYS = [+ 'SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED',+ 'SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS',+ 'LLM_CATALOG_SYNC_CRON_ENABLED',+] as const++type CronKey = (typeof CRON_KEYS)[number]++/** The code floor in `config.ts`; below it control-api refuses to boot. */+const MIN_INTERVAL_MS = 15 * 60_000++async function loadConfigWith(overrides: Partial<Record<CronKey, string>>) {+ const originalValues = new Map<string, string | undefined>()+ for (const key of CRON_KEYS) {+ originalValues.set(key, process.env[key])+ delete process.env[key]+ }+ Object.assign(process.env, overrides)+ vi.resetModules()+ try {+ const mod = await import('../src/config.js')+ return mod.config+ } finally {+ for (const key of CRON_KEYS) {+ const value = originalValues.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ }+}++function read(relativeFromThisFile: string): string {+ return readFileSync(new URL(relativeFromThisFile, import.meta.url), 'utf-8')+}++/** Fail-loud single-match extraction — a miss means the key is unregistered. */+function extractOne(source: string, pattern: RegExp, label: string): string {+ const match = source.match(pattern)+ if (!match || match[1] === undefined) {+ throw new Error(`Could not extract ${label} with ${pattern} — register the key`)+ }+ return match[1]+}++describe('subscription catalog sync cron config', () => {+ afterEach(() => {+ vi.resetModules()+ })++ it('defaults the reconciliation off and the interval to six hours', async () => {+ const config = await loadConfigWith({})++ expect(config.subscriptionCatalogSyncCronEnabled).toBe(false)+ expect(config.subscriptionCatalogSyncIntervalMs).toBe(6 * 60 * 60 * 1000)+ })++ it('honours an interval override at or above the floor', async () => {+ const config = await loadConfigWith({+ SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS: String(MIN_INTERVAL_MS),+ })++ expect(config.subscriptionCatalogSyncIntervalMs).toBe(MIN_INTERVAL_MS)+ })++ it('refuses to boot on an interval below the floor', async () => {+ // Fail loud, at import. A silently clamped interval would turn a typo into+ // one vendor call per connected grant every few seconds.+ await expect(+ loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS: String(MIN_INTERVAL_MS - 1) })+ ).rejects.toThrow(/SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS must be an integer >= 900000/)+ })++ it('turns the reconciliation on only for the exact string "true"', async () => {+ // Same exact-token idiom as the discovery cron: `enabled=1` or `yes` must+ // not silently start issuing vendor calls.+ expect(+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: 'true' }))+ .subscriptionCatalogSyncCronEnabled+ ).toBe(true)+ expect(+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: '1' }))+ .subscriptionCatalogSyncCronEnabled+ ).toBe(false)+ expect(+ (await loadConfigWith({ SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED: 'TRUE' }))+ .subscriptionCatalogSyncCronEnabled+ ).toBe(false)+ })++ it('registers all three cron keys in the base ConfigMap with the deployed values', async () => {+ const config = await loadConfigWith({})+ const source = read('../../deploy/base/control-plane/configmaps.yaml')++ // Off in the cluster, matching the code default: each tick calls the broker+ // once per connected grant, so this is turned on per deployment.+ expect(+ extractOne(+ source,+ /SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED:\s*"([^"]+)"/,+ 'SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED in deploy/base/control-plane/configmaps.yaml'+ )+ ).toBe('false')+ expect(config.subscriptionCatalogSyncCronEnabled).toBe(false)++ // Declared at the code default so the interval is visible where it is set.+ const interval = Number(+ extractOne(+ source,+ /SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS:\s*"(\d+)"/,+ 'SUBSCRIPTION_CATALOG_SYNC_INTERVAL_MS in deploy/base/control-plane/configmaps.yaml'+ )+ )+ expect(interval).toBe(config.subscriptionCatalogSyncIntervalMs)+ expect(interval).toBeGreaterThanOrEqual(MIN_INTERVAL_MS)++ // Deliberately NOT the code default: the discovery sync is one call to+ // models.dev per tick, so it runs everywhere.+ expect(+ extractOne(+ source,+ /LLM_CATALOG_SYNC_CRON_ENABLED:\s*"([^"]+)"/,+ 'LLM_CATALOG_SYNC_CRON_ENABLED in deploy/base/control-plane/configmaps.yaml'+ )+ ).toBe('true')+ expect(config.llmCatalogSyncCronEnabled).toBe(false)+ })+})diff --git a/control-api/test/routes.admin.codexSubscription.test.ts b/control-api/test/routes.admin.codexSubscription.test.tsindex e8a6e9e34..9e4b4b1c6 100644--- a/control-api/test/routes.admin.codexSubscription.test.ts+++ b/control-api/test/routes.admin.codexSubscription.test.ts@@ -439,7 +439,14 @@ describe('admin Codex subscription routes', () => {expect(res.status).toBe(503)expect(res.body.error).toBe('catalog_sync_failed')expect(JSON.stringify(res.body)).not.toContain('proxy down')- expect(materialize).toHaveBeenCalledTimes(1)+ // `never_synced` with no `persisted` flag means the row was not written, so+ // the ConfigMap carries nothing new. Report what the sync reported: calling+ // this `unavailable` would claim the vendor was reached and was down.+ expect(res.body.outcome).toBe('never_synced')+ // Liveness witness: the handler really ran the sync and really took the+ // failure path, so the absent publish is the rule and not an unreached one.+ expect(oauth.runCatalogSync).toHaveBeenCalledTimes(1)+ expect(materialize).not.toHaveBeenCalled()assertNoLeak(res.body)})diff --git a/control-api/test/routes.admin.grokSubscription.test.ts b/control-api/test/routes.admin.grokSubscription.test.tsindex b5aecace1..2d2cfbde6 100644--- a/control-api/test/routes.admin.grokSubscription.test.ts+++ b/control-api/test/routes.admin.grokSubscription.test.ts@@ -571,10 +571,12 @@ describe('admin Grok subscription routes', () => {expect(materialize).toHaveBeenCalledTimes(1)})- // `catalogStatus: 'never_synced'` is the service's own signal that NOTHING- // was persisted for the connection: no grant, a lost revision fence, a held- // refresh lock, an OAuth error before the catalog call. There is no new- // state for the ConfigMap to carry, so the endpoint must not write one.+ // `catalogStatus: 'never_synced'` WITHOUT `persisted` is the service's own+ // signal that nothing at all was written for the connection: no grant, a+ // lost revision fence, a held refresh lock, an OAuth error raised before+ // anything touched the row. There is no new state for the ConfigMap to+ // carry, so the endpoint must not write one. `never_synced` on its own does+ // NOT mean this — see the persisted case below.it.each([[{ reason: 'no_grant' }, 404, { error: 'no_grant' }],[{ reason: 'disabled' }, 404, { error: 'disabled' }],@@ -603,6 +605,32 @@ describe('admin Grok subscription routes', () => {})+ // The write landed and the sync still reports `never_synced`.+ // `markGrokRefreshSubjectMismatch` sets `status = 'reauth_required'` and+ // THEN throws, so the catalog was never synced while the CONNECTION row+ // changed. `llmAllowedModelsConfigMap` maps that status into the ConfigMap,+ // so the publish is owed before the 400 — and it is owed HERE, because no+ // later reconciliation repairs it: the cron skips a row that is no longer+ // `connected`.+ it('publishes a persisted status before the error response', async () => {+ const materialize = vi.fn(async () => {})+ grokOAuth.runGrokCatalogSync.mockResolvedValue({+ ok: false,+ catalogStatus: 'never_synced',+ reason: 'reauth_required',+ persisted: true,+ })+ const res = await request(makeApp(makeGateway(materialize))).post(+ `${GROK}/connections/team-grok/catalog/sync`+ )+ expect(res.status).toBe(400)+ expect(res.body).toEqual({ error: 'reauth_required' })+ // Liveness witness: the sync ran and failed with this reason, so the+ // publish below is the rule under test and not an unreached path.+ expect(grokOAuth.runGrokCatalogSync).toHaveBeenCalledTimes(1)+ expect(materialize).toHaveBeenCalledTimes(1)+ })+// A non-ready catalogStatus means the OPPOSITE: the sync reached xAI, the// answer was refused or unavailable, and the connection row now carries// that outcome. mcp-host and HCC never read Postgres, so withholding thediff --git a/control-ui/components/CodexSubscriptionHub/index.tsx b/control-ui/components/CodexSubscriptionHub/index.tsxindex 10f295834..f8ee9a0d5 100644--- a/control-ui/components/CodexSubscriptionHub/index.tsx+++ b/control-ui/components/CodexSubscriptionHub/index.tsx@@ -29,6 +29,7 @@ import {} from '@lib/codexSubscriptionFeature'import {GROK_DEVICE_VERIFICATION_ORIGIN,+ type GrokCatalogSyncView,type GrokSubscriptionConnectionView,createGrokSubscriptionConnection,isAllowedGrokVerificationUri,@@ -577,17 +578,25 @@ export function CodexSubscriptionHub() {setBusyKey(hubRowId(row))setError('')try {- const synced = grok- ? await syncGrokSubscriptionCatalog(row.connectionKey)- : await syncCodexSubscriptionCatalog(row.connectionKey)- const models = grok- ? await listGrokConnectionModels(row.connectionKey)- : await listCodexConnectionModels(row.connectionKey)- setEditModels(models)- if (synced.connection) setEditing(asHubRow(synced.connection, row.broker))- // A refresh failure after a landed sync is partial — the catalog did- // change, so it must not surface as "sync failed".- await load()+ // Only the sync call itself may report "sync failed". Everything after it+ // re-reads what the screen shows, and a failure there is partial: the+ // catalog on the server DID change. Sharing one `try` made a failed+ // re-read tell the operator their sync had failed, so they would sign in+ // again to repeat a sync that had already landed.+ // The narrower of the two views: the Hub reads only `outcome` and+ // `connection`, and the Codex view satisfies it.+ let synced: GrokCatalogSyncView+ try {+ synced = grok+ ? await syncGrokSubscriptionCatalog(row.connectionKey)+ : await syncCodexSubscriptionCatalog(row.connectionKey)+ } catch (err) {+ setError(catalogSyncErrorMessage(err, brand))+ return+ }++ // The outcome of the sync is reported before the re-read, so the operator+ // learns it whether or not the refresh below succeeds.if (synced.outcome === 'ready') {showToast('Catalog synced', { tone: 'success' })} else {@@ -595,8 +604,19 @@ export function CodexSubscriptionHub() {tone: 'error',})}- } catch (err) {- setError(catalogSyncErrorMessage(err, brand))++ try {+ const models = grok+ ? await listGrokConnectionModels(row.connectionKey)+ : await listCodexConnectionModels(row.connectionKey)+ setEditModels(models)+ if (synced.connection) setEditing(asHubRow(synced.connection, row.broker))+ await load()+ } catch {+ setError(+ `The ${brand} catalog was synced, but this view could not be refreshed. Reload the page to see it.`+ )+ }} finally {setBusyKey(null)}diff --git a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsxindex 77211eb34..616c32d7a 100644--- a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx+++ b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx@@ -930,6 +930,34 @@ describe('CodexSubscriptionHub catalog re-sync', () => {expect(await screen.findByText('Catalog synced')).toBeInTheDocument()})+ // The sync POST landed and the catalog on the server DID change. Only the+ // re-read of the models failed. Telling the operator "sync failed" here sends+ // them to sign in again and repeat a sync that already succeeded — and, on a+ // grant that rotates its refresh token, the repeat has a real cost.+ it('keeps a landed sync separate from a failed refresh of the view', async () => {+ vi.mocked(listGrokConnectionModels)+ .mockResolvedValueOnce([{ model: 'grok-4.6', enabled: true, stale: false }])+ .mockRejectedValueOnce(new Error('models listing failed'))+ fetchMock.mockResolvedValueOnce(+ makeResponse(200, {+ outcome: 'ready',+ connection: connection({ connectionKey: 'grok-aaa', displayName: 'Team Grok' }),+ })+ )++ const dialog = await openGrokDialog()+ await within(dialog).findByLabelText('grok-4.6')+ fireEvent.click(within(dialog).getByRole('button', { name: 'Sync catalog' }))++ // Liveness witness: the POST really went out and the re-read really ran and+ // really rejected, so the copy below is the rule and not an unreached path.+ await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1))+ await waitFor(() => expect(listGrokConnectionModels).toHaveBeenCalledTimes(2))+ expect(await screen.findByText('Catalog synced')).toBeInTheDocument()+ expect(screen.queryByText(/Catalog sync failed/)).not.toBeInTheDocument()+ expect(await screen.findByText(/could not be refreshed/)).toBeInTheDocument()+ })+it('reports a non-ready outcome the endpoint returns with 200 instead of claiming success', async () => {fetchMock.mockResolvedValueOnce(makeResponse(200, {diff --git a/control-ui/components/__tests__/LlmProviderConfig.test.tsx b/control-ui/components/__tests__/LlmProviderConfig.test.tsxindex 5f65db232..545947ab1 100644--- a/control-ui/components/__tests__/LlmProviderConfig.test.tsx+++ b/control-ui/components/__tests__/LlmProviderConfig.test.tsx@@ -193,13 +193,16 @@ describe('LlmProviderConfig (spec Topic 1b — domain projection + usable gate)'expect(screen.getByLabelText('Model', { selector: '#llm-primary-model' })).toBeInTheDocument()})- it('offers one OpenAI provider and does not invent a ChatGPT radio', () => {+ // This picker chooses the CREDENTIAL PATH for an agent, so the API key and+ // the subscription are peers and both appear. The /llm-models catalog is the+ // surface that shows one row per family, and it collapses its own rows.+ // Neither is a radio group: the credential path is a provider option, not a+ // second control beside it.+ it('offers the OpenAI API key and the OpenAI subscription as peer options', () => {render(<Harness />)fireEvent.click(screen.getByLabelText('Provider', { selector: '#llm-primary-provider' }))expect(screen.getByRole('option', { name: /^OpenAI$/i })).toBeInTheDocument()- expect(- screen.queryByRole('option', { name: /OpenAI Codex Subscription/i })- ).not.toBeInTheDocument()+ expect(screen.getByRole('option', { name: /OpenAI Codex Subscription/i })).toBeInTheDocument()expect(screen.queryByRole('radio', { name: /ChatGPT subscription/i })).not.toBeInTheDocument()expect(screen.queryByRole('radio', { name: /API key/i })).not.toBeInTheDocument()})diff --git a/control-ui/lib/__tests__/llm.test.ts b/control-ui/lib/__tests__/llm.test.tsindex 0f3a7d358..c75f12deb 100644--- a/control-ui/lib/__tests__/llm.test.ts+++ b/control-ui/lib/__tests__/llm.test.ts@@ -6,11 +6,9 @@ import {LLM_DEFAULT_MODEL_BY_PROVIDER,LLM_SECRET_EDITOR_GROUPS,type LlmModelCatalogEntry,- OPERATOR_PROVIDER_OPTIONS,brokerBackedRecipeAuthoringError,budgetUnitAllowedForProviders,catalogGroupKey,- familyCredentialSources,getAllModelOptions,getLlmGroupCompleteness,getModelOptions,@@ -19,6 +17,7 @@ import {inferProviderFromModels,isOpenAiFamily,llmChainRequiresSecret,+ operatorProviderOptions,providerRequiresLlmSecret,resolveDefaultModel,validateLlmSecretData,@@ -70,11 +69,25 @@ describe('getModelOptions', () => {})describe('OpenAI family presentation', () => {- it('does not offer Codex as a second provider in operator pickers', () => {- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'codex-subscription')).toBe(- false- )- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'openai')).toBe(true)+ // The agent-side credential picker offers a credential PATH, so an API key and+ // a subscription are peers there and both families are treated alike. One row+ // per family is the rule of the /llm-models catalog, which collapses its rows+ // with `collapseFamilyRows` and never calls this function.+ it('offers an API key and a subscription at the same level, in both families', () => {+ const options = operatorProviderOptions({ grokEnabled: true })+ expect(options.some(option => option.value === 'openai')).toBe(true)+ expect(options.some(option => option.value === 'codex-subscription')).toBe(true)+ expect(options.some(option => option.value === 'xai')).toBe(true)+ expect(options.some(option => option.value === 'grok-subscription')).toBe(true)+ })++ // The Grok filter is a deployment capability gate, not a statement about+ // families: Codex has no such signal in control-ui and is never gated.+ it('hides only Grok, and only until the deployment gate is proven on', () => {+ const gated = operatorProviderOptions()+ expect(gated.some(option => option.value === 'grok-subscription')).toBe(false)+ expect(gated.some(option => option.value === 'codex-subscription')).toBe(true)+ expect(gated.some(option => option.value === 'xai')).toBe(true)})it('labels the subscription runtime id as OpenAI for operators', () => {@@ -95,9 +108,6 @@ describe('OpenAI family presentation', () => {expect(catalogGroupKey('xai')).toBe('xai')expect(isOpenAiFamily('grok-subscription')).toBe(false)expect(getProviderLabel('grok-subscription')).not.toBe('OpenAI')- expect(OPERATOR_PROVIDER_OPTIONS.some(option => option.value === 'grok-subscription')).toBe(- true- )})it('keeps the broker label distinct from its family label', () => {@@ -113,42 +123,6 @@ describe('OpenAI family presentation', () => {// not be folded into anything.expect(catalogGroupKey('not-a-provider')).toBe('not-a-provider')})-- it('marks a model as API key, subscription, or both, in any family', () => {- const catalog: LlmModelCatalogEntry[] = [- { provider: 'openai', model: 'gpt-5.1', enabled: true },- { provider: 'codex-subscription', model: 'gpt-5.1', enabled: true },- { provider: 'codex-subscription', model: 'gpt-5.3-codex', enabled: true },- { provider: 'xai', model: 'grok-4.6', enabled: true },- { provider: 'grok-subscription', model: 'grok-4.6', enabled: true },- { provider: 'grok-subscription', model: 'grok-4.6-fast', enabled: true },- ]- expect(familyCredentialSources(catalog, 'openai', 'gpt-5.1')).toEqual({- apiKey: true,- subscription: true,- })- expect(familyCredentialSources(catalog, 'openai', 'gpt-5.3-codex')).toEqual({- apiKey: false,- subscription: true,- })- expect(familyCredentialSources(catalog, 'xai', 'grok-4.6')).toEqual({- apiKey: true,- subscription: true,- })- expect(familyCredentialSources(catalog, 'xai', 'grok-4.6-fast')).toEqual({- apiKey: false,- subscription: true,- })- // A family does not see another family's rows.- expect(familyCredentialSources(catalog, 'openai', 'grok-4.6')).toEqual({- apiKey: false,- subscription: false,- })- expect(familyCredentialSources(catalog, 'not-a-provider', 'grok-4.6')).toEqual({- apiKey: false,- subscription: false,- })- })})describe('getAllModelOptions', () => {diff --git a/control-ui/lib/grokSubscription.ts b/control-ui/lib/grokSubscription.tsindex 5b82d68db..95f4c6e17 100644--- a/control-ui/lib/grokSubscription.ts+++ b/control-ui/lib/grokSubscription.ts@@ -23,7 +23,14 @@ export type GrokCatalogStatus = CodexCatalogStatusexport type GrokConnectionStatus = CodexConnectionStatusexport type GrokOAuthIntent = CodexOAuthIntentexport type GrokDevicePollView = CodexDevicePollView-export type GrokCatalogSyncView = CodexCatalogSyncView+/**+ * Narrower than the Codex view on purpose. The Grok `catalog/sync` endpoint+ * answers `{ outcome, connection }` and sends no `added`/`refreshed`/`staled`+ * counters, so `sanitizeCodexCatalogSync` fills them with zeros that mean "the+ * endpoint said nothing", not "nothing changed". Promising the fields would+ * invite a caller to render those zeros as a result.+ */+export type GrokCatalogSyncView = Omit<CodexCatalogSyncView, 'added' | 'refreshed' | 'staled'>export type GrokDeviceStartView = {userCode: stringdiff --git a/control-ui/lib/llm.ts b/control-ui/lib/llm.tsindex 7e45ec589..78037b3a6 100644--- a/control-ui/lib/llm.ts+++ b/control-ui/lib/llm.ts@@ -5,7 +5,6 @@ import {PROVIDER_DISPLAY_LABELS,PROVIDER_IDS,PROVIDER_NON_SECRET_ENV,- familyProviderIds,isLlmProviderId,providerFamily,} from '@clerum/llm-providers'@@ -51,7 +50,6 @@ export const LLM_PROVIDER_OPTIONS: Array<{ value: LlmProvider; label: string }>id => ({ value: id, label: PROVIDER_DISPLAY_LABELS[id] }))-/** Runtime broker id. Never offer this as a second Provider row in operator pickers. */export const OPENAI_SUBSCRIPTION_PROVIDER = 'codex-subscription' as constexport const GROK_SUBSCRIPTION_PROVIDER = 'grok-subscription' as const@@ -69,15 +67,25 @@ export function isOauthBrokerProvider(provider: string | undefined | null): bool)}-/** Provider dropdown: one OpenAI entry. Runtime still persists `codex-subscription`. */-export const OPERATOR_PROVIDER_OPTIONS: Array<{ value: LlmProvider; label: string }> =- LLM_PROVIDER_OPTIONS.filter(option => option.value !== OPENAI_SUBSCRIPTION_PROVIDER)--/** Hide Grok until Control API proves the flag is on. */+/**+ * The agent-side credential picker (`LlmPolicyEditor`, `LlmProviderConfig`):+ * every provider id, API keys and subscription brokers at the same level,+ * because that is the choice being made — which credential path this agent+ * uses. Both families are treated identically: `openai` beside+ * `codex-subscription`, `xai` beside `grok-subscription`.+ *+ * One row per family is the rule of a DIFFERENT surface — the `/llm-models`+ * catalog, where `collapseFamilyRows` groups the models of a family under one+ * provider row with an `API key · Subscription` badge. That surface does not+ * call this function.+ *+ * The only filter left is the Grok deployment gate: a capability the Control+ * API has to confirm, not a statement about families.+ */export function operatorProviderOptions(opts?: {grokEnabled?: boolean}): Array<{ value: LlmProvider; label: string }> {- return OPERATOR_PROVIDER_OPTIONS.filter(+ return LLM_PROVIDER_OPTIONS.filter(option => option.value !== GROK_SUBSCRIPTION_PROVIDER || opts?.grokEnabled === true)}@@ -113,33 +121,6 @@ export function catalogGroupKey(provider: string): string {return isLlmProviderId(provider) ? providerFamily(provider) : provider}-export type FamilyCredentialSource = 'api-key' | 'subscription'--/**- * Which credential paths serve a model inside one family. Derived from each- * provider's auth mode rather than named ids, so a family gains a broker without- * this function changing. An unknown family has no members, so neither path- * serves it.- */-export function familyCredentialSources(- catalog: LlmModelCatalogEntry[],- family: string,- model: string-): { apiKey: boolean; subscription: boolean } {- const members = familyProviderIds(family)- const served = (mode: (typeof PROVIDER_AUTH_MODE)[LlmProviderId]): boolean =>- catalog.some(- row =>- row.model === model &&- row.enabled &&- !row.stale &&- isLlmProviderId(row.provider) &&- members.includes(row.provider) &&- PROVIDER_AUTH_MODE[row.provider] === mode- )- return { apiKey: served('static-credentials'), subscription: served('oauth-broker') }-}-// The list of usable models per provider is no longer a static catalog: it is// the operator-declared allowlist served by control-api (`/admin/llm-models`,// spec §3-R3). Fetch it with `useLlmAllowedModels()` and pass the rows to thediff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 8936b7013..7118ebe17 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.606",+ "version": "0.1.607","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.606",+ "version": "0.1.607","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 93db41c74..d4848f328 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.606",+ "version": "0.1.607","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/packages/llm-providers/index.d.ts b/packages/llm-providers/index.d.tsindex ccb5508e4..99c20f0db 100644--- a/packages/llm-providers/index.d.ts+++ b/packages/llm-providers/index.d.ts@@ -150,8 +150,12 @@ export declare function buildProviderMaps(PROVIDER_AUTH_MODE: Readonly<Record<string, ProviderAuthMode>>PROVIDER_MODEL_CATALOG_MODE: Readonly<Record<string, ProviderModelCatalogMode>>}-/** Every provider id → the provider id that owns its family. Total, frozen. */-export declare const PROVIDER_FAMILY: Record<LlmProviderId, LlmProviderId>+/**+ * Every provider id → the provider id that owns its family. Total, frozen.+ * `Readonly` because the value really is `Object.freeze`d: a plain `Record`+ * type-checks a write that then silently no-ops at runtime.+ */+export declare const PROVIDER_FAMILY: Readonly<Record<LlmProviderId, LlmProviderId>>/** The family of a known provider. Throws on an unrecognised id. */export declare function providerFamily(id: LlmProviderId): LlmProviderId/**diff --git a/scripts/e2e/e2e-codex-subscription-playwright.sh b/scripts/e2e/e2e-codex-subscription-playwright.shindex dc693e7e8..5ec33b559 100755--- a/scripts/e2e/e2e-codex-subscription-playwright.sh+++ b/scripts/e2e/e2e-codex-subscription-playwright.sh@@ -147,6 +147,29 @@ fiwait_http control-ui "${CONTROL_UI_URL%/}/" 90wait_http control-api "${CONTROL_API_URL%/}/health" 60+# Fail-closed deployment preconditions for the catalog re-sync guardian.+#+# control-api collapses "proxy admin URL unset", "proxy unreachable" and "the+# vendor answered badly" into one identical response — 503, catalog_sync_failed,+# outcome `unavailable` — so no assertion in the spec can separate them. A lane+# missing either precondition would render an error banner and+# codex-subscription-catalog-resync.spec.ts would pass its 503 branch while no+# operator could ever sync a catalog. Checked here, before a browser starts,+# where the diagnosis is unambiguous and costs two API reads.+log "Verifying Codex catalog sync deployment preconditions"+proxy_admin_url="$(kubectl --context="${PROFILE}" -n control-plane get configmap control-api-config \+ -o jsonpath='{.data.CODEX_LLM_PROXY_ADMIN_URL}')" \+ || die "cannot read configmap/control-api-config in namespace control-plane"+[[ -n "${proxy_admin_url}" ]] \+ || die "CODEX_LLM_PROXY_ADMIN_URL is empty in configmap/control-api-config; control-api would answer every catalog sync with 503 unavailable without ever reaching the proxy"++proxy_ready="$(kubectl --context="${PROFILE}" -n control-plane get deployment codex-llm-proxy \+ -o jsonpath='{.status.readyReplicas}')" \+ || die "cannot read deployment/codex-llm-proxy in namespace control-plane"+[[ "${proxy_ready:-0}" -ge 1 ]] \+ || die "deployment/codex-llm-proxy has no ready replica (readyReplicas='${proxy_ready:-0}'); control-api would answer every catalog sync with 503 unavailable"+log "Preconditions OK (proxy admin URL set, codex-llm-proxy readyReplicas=${proxy_ready})"+log "Running Control UI Playwright guardians (CONTROL_UI_URL=${CONTROL_UI_URL})"(cd "${PLAYWRIGHT_DIR}"diff --git a/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.tsindex 2f373740a..f7eb6c939 100644--- a/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts+++ b/tests/e2e/playwright/control-ui/codex-subscription-catalog-resync.spec.ts@@ -13,9 +13,17 @@* for the rows and the union allowlist.** What this lane uniquely proves: the affordance exists on a connected grant- * and nowhere else, the click really issues the write, and the window settles- * on the answer the API gave instead of leaving the operator with a spinner- * and no verdict.+ * and nowhere else, the click really issues the write, the write is RECORDED,+ * and the window settles on the answer the API gave instead of leaving the+ * operator with a spinner and no verdict.+ *+ * Its honest limit: control-api collapses "proxy admin URL unset", "proxy+ * unreachable" and "chatgpt.com answered badly" into the same+ * `503 / catalog_sync_failed / unavailable`, so no assertion at this lane's+ * HTTP surface can separate a vendor outage from a broken deployment. Those two+ * are deployment preconditions and are checked fail-closed by the lane runner+ * (`scripts/e2e/e2e-codex-subscription-playwright.sh`) BEFORE Playwright+ * starts, which is why the 503 branch below can stay green on `unavailable`.** Auth: `/` + loginControlUiVisible. No terminal `goto`. No test.skip.*/@@ -27,11 +35,29 @@ import { ControlUiShell, SecretsLlmSubscriptionsPage } from '../pages/codex-subsconst SYNC_PATH =/\/api\/v1\/admin\/llm\/providers\/codex-subscription\/connections\/[^/]+\/catalog\/sync$/-// The closed set the route answers with (control-api/src/routes/admin/-// codexSubscription.ts:734-770). A status outside it means the contract moved-// and the mapping in CodexSubscriptionHub is stale, which is worth failing on-// even though the operator-visible outcome would still be "an error banner".-const DOCUMENTED_SYNC_STATUS = [200, 401, 403, 404, 409, 503]+// The closed set of outcomes a CORRECTLY PROVISIONED lane can answer with+// (control-api/src/routes/admin/codexSubscription.ts). A status outside it+// means either the contract moved or this lane's own environment is broken,+// and both must be red.+//+// What is deliberately absent, and why an error banner alone is not enough to+// pass:+// - 401/403: the lane's admin session was refused. Not an outcome of this+// feature at all.+// - 404: `disabled` is unreachable because the list route answers 404 too, so+// `listCodexConnections()` would have thrown before the click; `no_grant`+// means the connection row exists without its secrets, i.e. lane state+// corruption.+// - 503 with any outcome other than `unavailable`: `auth-rejected` (dead grant+// token or a broken proxy permit), `never_synced` (row disconnected under+// us, DB error) and `configmap_write_failed` are all this lane's own+// breakage wearing a product-shaped banner.+const DOCUMENTED_SYNC_STATUS = [200, 409, 503]++// One spelling, shared by the affordance assertion in R1 and the ABSENCE+// assertion in R2. Two literals would let a rename leave R2 asserting that a+// button nobody renders is not rendered — green, and proving nothing.+const SYNC_BUTTON = 'Sync catalog'async function loginFromHome(page: Page) {await page.goto('/')@@ -77,7 +103,7 @@ test.describe('Codex subscription catalog re-sync', () => {await modelsGetconst dialog = page.getByRole('dialog')- const syncButton = dialog.getByRole('button', { name: 'Sync catalog' })+ const syncButton = dialog.getByRole('button', { name: SYNC_BUTTON })await expect(syncButton).toBeVisible()await expect(syncButton).toBeEnabled()@@ -99,25 +125,48 @@ test.describe('Codex subscription catalog re-sync', () => {const errorBanner = dialog.locator('.cu-banner--error')if (syncRes.status() === 200) {+ // `ok: true` carries `catalogStatus: 'ready'` and nothing else+ // (codexSubscriptionOAuth.ts CodexCatalogSyncResult), so a 200 with any+ // other outcome is a contract move, not a degraded sync. The old+ // non-ready arm here was unreachable.const body = (await syncRes.json()) as { outcome?: string }+ expect(body.outcome).toBe('ready')// A recorded outcome always advances catalog_revision// (codexSubscriptionConnection.ts:692), so this is the business signal// that the write landed rather than the UI merely redrawing.expect(await catalogRevisionOf(grantKey)).toBeGreaterThan(revisionBefore)- if (body.outcome === 'ready') {- await expect(successToast).toBeVisible()- await expect(errorBanner).toHaveCount(0)- } else {- await expect(- page.getByRole('status').filter({ hasText: 'Catalog sync failed' })- ).toBeVisible()- }- } else {- // The operator must be told. An empty banner is the failure this guards:- // the modal would otherwise look identical to a successful sync.+ await expect(successToast).toBeVisible()+ await expect(errorBanner).toHaveCount(0)+ } else if (syncRes.status() === 409) {+ // A concurrent writer won the revision or refresh-lock fence — the+ // reconciliation cron runs the very same runCodexCatalogSync on Codex+ // grants, so it can race this click. The click still issued the write;+ // this request recorded nothing, so the revision is NOT asserted.+ const body = (await syncRes.json()) as { error?: string }+ expect(['stale_revision', 'refresh_in_flight']).toContain(body.error)await expect(errorBanner).toBeVisible()await expect(errorBanner).not.toHaveText('')await expect(successToast).toHaveCount(0)+ } else {+ // 503 is accepted for exactly one outcome: the vendor answered badly.+ // That is the only non-200 this lane cannot control, because the catalog+ // origin is frozen to chatgpt.com by codex-llm-proxy/src/originPolicy.ts+ // and this lane deploys no fake upstream.+ expect(syncRes.status()).toBe(503)+ const body = (await syncRes.json()) as { error?: string; outcome?: string }+ expect(body.error).toBe('catalog_sync_failed')+ expect(body.outcome).toBe('unavailable')+ // The outcome was RECORDED, so the revision advanced: the sync ran end to+ // end and only the upstream failed. A transport that never ran — a dead+ // grant token, a disconnected row, a failed ConfigMap write — cannot+ // produce this, which is what keeps the branch from passing on breakage.+ expect(await catalogRevisionOf(grantKey)).toBeGreaterThan(revisionBefore)+ // The operator must be told, and told WHAT: an empty or generic banner is+ // the failure this guards, because the modal would otherwise look+ // identical to a successful sync.+ await expect(errorBanner).toBeVisible()+ await expect(errorBanner).toContainText('unavailable')+ await expect(successToast).toHaveCount(0)}// Whatever the outcome, the control comes back: a stuck busy state would@@ -145,7 +194,7 @@ test.describe('Codex subscription catalog re-sync', () => {expect(created, 'the grant under test must be listed').toBeTruthy()expect(created!.status).not.toBe('connected')- await expect(dialog.getByRole('button', { name: 'Sync catalog' })).toHaveCount(0)+ await expect(dialog.getByRole('button', { name: SYNC_BUTTON })).toHaveCount(0)await hub.closeConnectModal()})diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex 7acdea7b6..d921d1cec 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.63",+ "version": "0.1.64","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.63",+ "version": "0.1.64","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0",diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 8260c6483..d7b65879c 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.63",+ "version": "0.1.64","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {