Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Subscription catalog reconciliation cron.
*
* A subscription grant's catalog (`codex_catalog_models`, `grok_catalog_models`)
* is written during the OAuth handshake and never again on its own. A model the
* vendor publishes afterwards therefore stays invisible to that connection —
* while the API-key provider of the SAME vendor picks it up from the discovery
* sync — until an operator asks for it. This cron closes that gap by re-running
* the very same `runCodexCatalogSync` / `runGrokCatalogSync` the connect flow
* and the manual `POST .../catalog/sync` endpoint drive. No HTTP hop, no new
* auth, no second write path.
*
* Hazard handling follows `llmCatalogSyncCron.ts`, which is the reference
* implementation for every one of them:
*
* - Cross-replica dedup through a SESSION-scoped
* `pg_try_advisory_lock(hashtext('subscription-catalog-sync-cron-v1'))` on a
* dedicated client. A session lock is NOT released by COMMIT nor by returning
* the connection to the pool, so the `finally` unlocks explicitly and, when
* the unlock itself fails, DESTROYS the connection (`release(err)`) — the only
* remaining way to end the session and free the lock. A leaked lock would
* wedge every future tick.
* - First tick after random jitter, then `setInterval`; both handles `unref()`'d,
* start is not awaited by boot, and errors are logged but never thrown.
* - The reconciliation closure is injected by `main.ts`, because publishing the
* allowlist ConfigMap needs the K8s gateway and this module owns none.
*
* Cost discipline: every tick issues one upstream catalog call PER CONNECTION,
* and a Grok refresh inside the 5-minute skew ROTATES the refresh token. A short
* interval is an abuse risk, not a freshness win — hence the conservative
* default and the floor enforced in `config.ts`.
*/
import { config } from '../config.js'
import { pool } from '../db.js'
import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'
import { rootLogger } from '../observability/logger.js'
import { createCodexCatalogTransportFromEnv } from './codexSubscriptionCatalog.js'
import { listLiveCodexSubscriptionConnections } from './codexSubscriptionConnection.js'
import { runCodexCatalogSync } from './codexSubscriptionOAuth.js'
import {
buildCodexBrowserRedirectUri,
resolveCodexControlUiBaseUrl,
} from './codexSubscriptionRedirectUri.js'
import { createGrokCatalogTransportFromEnv } from './grokSubscriptionCatalog.js'
import {
assertGrokConnectionKey,
listLiveGrokSubscriptionConnections,
} from './grokSubscriptionConnection.js'
import { runGrokCatalogSync } from './grokSubscriptionOAuth.js'
import type { AllowedModelsConfigMapMaterializer } from './llmAllowedModelsConfigMap.js'
import {
publishAllowedModelsConfigMapAfterGrantChange,
syncOutcomeChangedTheRow,
} from './llmAllowedModelsConfigMap.js'
const log = rootLogger.child({ service: 'subscription_catalog_sync_cron' })
// Session-scoped advisory lock key, distinct from every other cron's. The `-v1`
// suffix leaves room to re-key if the tick's semantics ever change.
const CRON_LOCK_KEY_SQL = "hashtext('subscription-catalog-sync-cron-v1')"
export type SubscriptionBroker = 'codex-subscription' | 'grok-subscription'
/**
* What `runCodexCatalogSync` / `runGrokCatalogSync` return, reduced to the
* fields this module decides on.
*
* `catalogStatus` describes the CATALOG, so it cannot answer "did the connection
* row change?" on its own: a rejected Grok refresh token writes
* `status = 'reauth_required'` and only then throws, leaving the catalog at
* `never_synced` on a row that did change. `persisted` is the services' explicit
* answer for that case, and `syncOutcomeChangedTheRow` is the single place that
* reads both.
*/
export type SubscriptionCatalogSyncOutcome =
| { ok: true; catalogStatus: 'ready' }
| { ok: false; catalogStatus: string; reason?: string; persisted?: boolean }
export type SubscriptionConnectionRow = { connectionKey: string; status: string }
/** One broker's side of a tick. Production builds it from config + db + transport. */
export interface SubscriptionBrokerPort {
broker: SubscriptionBroker
/** The deployment gate (`config.*SubscriptionEnabled`). Off = the broker is dark. */
enabled: boolean
listConnections(): Promise<SubscriptionConnectionRow[]>
syncCatalog(connectionKey: string): Promise<SubscriptionCatalogSyncOutcome>
}
export interface SubscriptionCatalogReconcileDeps {
brokers: SubscriptionBrokerPort[]
/** Undefined = no ConfigMap writer, so the publish is a local no-op (unit tests). */
materializer: AllowedModelsConfigMapMaterializer | undefined
}
export interface SubscriptionCatalogReconcileResult {
/** Connections whose catalog is now `ready`. */
synced: number
/** Connections that RECORDED a non-ready outcome (`auth-rejected`, `unavailable`). */
degraded: number
/** Raced or lock-held; nothing recorded, resolves on the next tick. */
raced: number
failed: number
/** Gate off, not addressable, or `status !== 'connected'`. */
skipped: number
published: 'published' | 'skipped'
}
/**
* Transient, not failures. `stale_revision` means a concurrent manual sync won
* the `credential_revision AND catalog_revision` fence — its write landed, ours
* did not, and the catalog is fresh either way. `refresh_in_flight` means the
* per-connection 30s refresh lock was held. Both clear by themselves.
*/
const RACED_REASONS = new Set(['stale_revision', 'refresh_in_flight'])
/**
* Migration 0113_grok_subscription_terminal_connection_key archives superseded
* tombstones as `<key>~revoked~<id>`, outside the key grammar. `listLive*`
* filters `revoked_at IS NULL`, but an archived row can still surface here; a
* sync call on it would only raise `invalid_connection_key`. The admin router
* applies the same filter for the same reason.
*/
export function filterAddressableGrokConnections<T extends SubscriptionConnectionRow>(
rows: T[]
): T[] {
return rows.filter(row => {
try {
assertGrokConnectionKey(row.connectionKey)
return true
} catch {
return false
}
})
}
/**
* Reconcile every live, connected grant of every enabled broker, then publish
* the runtime allowlist ONCE.
*
* A per-connection failure is isolated: one rejected grant must not cost the
* others their sync. The publish, in contrast, is not isolated — a catalog that
* changed in Postgres but not in the ConfigMap is a divergence the runtime hosts
* would serve until the next mutation, so the rejection propagates and the tick
* reports `errored`. It runs on every tick regardless of what the loop recorded,
* which is what makes the next tick repair a publish that threw; see the comment
* at the publish itself.
*/
export async function reconcileSubscriptionCatalogs(
deps: SubscriptionCatalogReconcileDeps
): Promise<SubscriptionCatalogReconcileResult> {
let synced = 0
let degraded = 0
let raced = 0
let failed = 0
let skipped = 0
for (const port of deps.brokers) {
if (!port.enabled) {
log.debug(
{ event: 'subscription_catalog_sync_broker_disabled', broker: port.broker },
`${port.broker} is disabled for this deployment; skipping`
)
continue
}
// Outside the per-connection `try` below, so without this guard a single
// broker's listing failure would abandon the OTHER broker's connections
// too — the tick would throw out of the loop entirely. One failed listing
// is one failure, not a lost tick.
let rows: SubscriptionConnectionRow[]
try {
rows = await port.listConnections()
} catch (err) {
failed += 1
log.warn(
{
event: 'subscription_catalog_sync_listing_failed',
broker: port.broker,
err: err instanceof Error ? err.message : String(err),
},
`${port.broker} connection listing failed; continuing with the remaining brokers`
)
continue
}
for (const row of rows) {
// Every narrower rule is wrong: `disconnected` makes the catalog sync
// throw `not_connected`, `connecting` throws `no_grant` inside the
// ensure-fresh path, and `reauth_required` needs a human this cron
// cannot summon.
if (row.status !== 'connected') {
skipped += 1
continue
}
try {
const outcome = await port.syncCatalog(row.connectionKey)
if (outcome.ok) {
synced += 1
} else if (syncOutcomeChangedTheRow(outcome)) {
degraded += 1
log.info(
{
event: 'subscription_catalog_sync_degraded',
broker: port.broker,
connectionKey: row.connectionKey,
catalogStatus: outcome.catalogStatus,
// Present when the catalog stayed at `never_synced` and the
// CONNECTION row is what changed — the reason is then the only
// field that says what the row now carries.
reason: outcome.reason,
},
`${port.broker} catalog recorded a non-ready outcome`
)
} else if (outcome.reason && RACED_REASONS.has(outcome.reason)) {
raced += 1
} else {
failed += 1
log.warn(
{
event: 'subscription_catalog_sync_failed',
broker: port.broker,
connectionKey: row.connectionKey,
reason: outcome.reason,
},
`${port.broker} catalog sync recorded nothing`
)
}
} catch (err) {
failed += 1
log.warn(
{
event: 'subscription_catalog_sync_threw',
broker: port.broker,
connectionKey: row.connectionKey,
err: err instanceof Error ? err.message : String(err),
},
`${port.broker} catalog sync threw; continuing with the remaining connections`
)
}
}
}
// Publish on EVERY tick that ran, not only on one that wrote a row.
//
// `materialize()` rebuilds the ConfigMap from Postgres — the allowlist plus
// both brokers' live connections and their readiness — so the write is
// idempotent and publishing unconditionally costs one API call per interval.
// That is what lets a publish that threw converge on the next tick with no
// state carried across ticks or replicas: the advisory lock is taken per
// tick, so a flag set by the replica that failed would be invisible to
// whichever replica wins the next one.
//
// A "publish only when something was recorded" gate cannot do that job. The
// rows a failed tick changed are `reauth_required` by the next tick, the
// loop above skips them by status, nothing is recorded, and the stale
// ConfigMap — where mcp-host and HCC still see a usable grant — survives
// until an unrelated mutation or a control-api restart happens to republish.
//
// `llmAllowedModelsBootReconcile.ts` already writes unconditionally at boot
// for the same anti-drift reason; this makes the cron agree with it.
const published = await publishAllowedModelsConfigMapAfterGrantChange(deps.materializer)
return { synced, degraded, raced, failed, skipped, published }
}
function cronDbClient() {
return { query: (text: string, values?: unknown[]) => pool.query(text, values) }
}
/**
* Production wiring for both brokers, built at TICK time rather than at module
* load or boot.
*
* The laziness is load-bearing for Codex: `normalizeControlUiOrigin` throws on a
* malformed, non-http or path-bearing `CONTROL_API_CONTROL_UI_BASE_URL`, and a
* throw at boot would take control-api down instead of failing one tick.
*
* The redirect URI is built inside `syncCatalog`, not here. Building it while
* WIRING the ports would throw before the Grok port exists, so a Codex-only
* misconfiguration would cost Grok its entire reconciliation — the failure this
* function is meant to contain. Inside `syncCatalog` the throw is caught by the
* per-connection `try` in `reconcileSubscriptionCatalogs` and counted as one
* failed Codex connection, which is what it is.
*
* `runCodexCatalogSync` reaches the refresh path only — `exchangeRefreshToken`
* sends no `redirect_uri` — but a real value is built anyway rather than a
* placeholder that would be wrong the day the path changes.
*/
export function createSubscriptionBrokerPorts(): SubscriptionBrokerPort[] {
const db = cronDbClient()
const encryptionKey = deriveOAuthEncryptionKey(config.oauthEncryptionKey)
const codexTransport = createCodexCatalogTransportFromEnv()
const codex: SubscriptionBrokerPort = {
broker: 'codex-subscription',
enabled: config.codexSubscriptionEnabled,
listConnections: () => listLiveCodexSubscriptionConnections(db),
syncCatalog: connectionKey =>
runCodexCatalogSync(
{
db,
encryptionKey,
fetchFn: fetch,
clientId: config.codexOAuthClientId,
redirectUri: buildCodexBrowserRedirectUri(
resolveCodexControlUiBaseUrl(config.controlUiBaseUrl, undefined)
),
enabled: config.codexSubscriptionEnabled,
connectionKey,
},
connectionKey,
codexTransport
),
}
const grokTransport = createGrokCatalogTransportFromEnv()
const grok: SubscriptionBrokerPort = {
broker: 'grok-subscription',
enabled: config.grokSubscriptionEnabled,
listConnections: async () =>
filterAddressableGrokConnections(await listLiveGrokSubscriptionConnections(db)),
syncCatalog: connectionKey =>
runGrokCatalogSync(
{
db,
encryptionKey,
fetchFn: fetch,
clientId: config.grokOAuthClientId,
enabled: config.grokSubscriptionEnabled,
connectionKey,
},
connectionKey,
grokTransport
),
}
return [codex, grok]
}
/**
* The closure `main.ts` hands to the cron. It owns no gateway, so the caller
* passes the materializer the admin routes already use.
*/
export function reconcileSubscriptionCatalogsFromEnv(
materializer: AllowedModelsConfigMapMaterializer | undefined
): Promise<SubscriptionCatalogReconcileResult> {
return reconcileSubscriptionCatalogs({ brokers: createSubscriptionBrokerPorts(), materializer })
}
/**
* Minimal client shape for the advisory-lock session. pg's PoolClient satisfies
* it. `release` takes pg's optional destroy argument: a truthy value (an Error)
* DESTROYS the connection instead of returning it to the pool.
*/
type LockClient = {
query: (text: string, values?: unknown[]) => Promise<{ rows: unknown[]; rowCount: number | null }>
release: (destroy?: Error | boolean) => void
}
export interface SubscriptionCatalogSyncCronDeps {
connector?: { connect: () => Promise<LockClient> }
/** Injected by `main.ts` so the reconciliation can reach the gateway's materializer. */
sync: () => Promise<SubscriptionCatalogReconcileResult>
}
export interface SubscriptionCatalogSyncTickResult {
skippedLock: boolean
ran: boolean
errored: boolean
}
/**
* One tick: acquire the session advisory lock, reconcile, release. Never throws
* — a failure is logged and reported in the result. The lock is released in
* `finally` whether the tick ran, skipped, or failed.
*/
export async function runSubscriptionCatalogSyncTick(
deps: SubscriptionCatalogSyncCronDeps
): Promise<SubscriptionCatalogSyncTickResult> {
const connector = deps.connector ?? pool
let lockClient: LockClient | undefined
let locked = false
try {
lockClient = await connector.connect()
const lockRes = await lockClient.query(
`SELECT pg_try_advisory_lock(${CRON_LOCK_KEY_SQL}) AS acquired`
)
locked = (lockRes.rows[0] as { acquired?: boolean } | undefined)?.acquired === true
if (!locked) {
log.debug(
{ event: 'subscription_catalog_sync_cron_skipped_lock' },
'subscription catalog sync tick skipped: advisory lock held by another replica'
)
return { skippedLock: true, ran: false, errored: false }
}
const result = await deps.sync()
log.info(
{
event: 'subscription_catalog_sync_cron_tick',
synced: result.synced,
degraded: result.degraded,
raced: result.raced,
failed: result.failed,
skipped: result.skipped,
published: result.published,
},
`subscription catalog sync tick complete (synced=${result.synced}, degraded=${result.degraded}, raced=${result.raced}, failed=${result.failed}, skipped=${result.skipped}, ${result.published})`
)
return { skippedLock: false, ran: true, errored: false }
} catch (err) {
log.error(
{
event: 'subscription_catalog_sync_cron_error',
err: err instanceof Error ? err.message : String(err),
},
'subscription catalog sync tick failed'
)
return { skippedLock: false, ran: false, errored: true }
} finally {
if (lockClient) {
let destroyed = false
if (locked) {
try {
await lockClient.query(`SELECT pg_advisory_unlock(${CRON_LOCK_KEY_SQL})`)
} catch (unlockErr) {
// The session lock is freed only by an explicit unlock or by CLOSING
// the session. The unlock failed, so destroy the connection: a plain
// release() would hand a live session still holding the lock back to
// the pool and wedge every future tick.
const asError = unlockErr instanceof Error ? unlockErr : new Error(String(unlockErr))
log.warn(
{ event: 'subscription_catalog_sync_cron_unlock_failed', err: asError.message },
'pg_advisory_unlock failed; destroying the connection to release the session lock'
)
lockClient.release(asError)
destroyed = true
}
}
if (!destroyed) lockClient.release()
}
}
}
let intervalHandle: ReturnType<typeof setInterval> | null = null
let firstRunHandle: ReturnType<typeof setTimeout> | null = null
/**
* Upper bound of the random delay before the first tick. Keeps the replicas of
* one rollout from contending for the advisory lock at the same instant; the
* lock already makes a collision a harmless no-op.
*/
export const SUBSCRIPTION_CATALOG_SYNC_FIRST_RUN_JITTER_MS = 5_000
export function startSubscriptionCatalogSyncCron(
deps: SubscriptionCatalogSyncCronDeps,
intervalMs: number
): void {
if (intervalHandle || firstRunHandle) return
const run = (): void => {
// runSubscriptionCatalogSyncTick never rejects; the catch is defensive so an
// unexpected throw can never kill the timer.
void runSubscriptionCatalogSyncTick(deps).catch(err => {
log.error(
{
event: 'subscription_catalog_sync_cron_unhandled',
err: err instanceof Error ? err.message : String(err),
},
'unhandled error in subscription catalog sync cron'
)
})
}
const firstDelay = Math.floor(Math.random() * SUBSCRIPTION_CATALOG_SYNC_FIRST_RUN_JITTER_MS)
firstRunHandle = setTimeout(() => {
firstRunHandle = null
run()
intervalHandle = setInterval(run, intervalMs)
intervalHandle.unref()
}, firstDelay)
firstRunHandle.unref()
log.info(
{ event: 'subscription_catalog_sync_cron_started', intervalMs, firstRunInMs: firstDelay },
'subscription catalog sync cron started'
)
}
export function stopSubscriptionCatalogSyncCron(): void {
if (firstRunHandle) {
clearTimeout(firstRunHandle)
firstRunHandle = null
}
if (intervalHandle) {
clearInterval(intervalHandle)
intervalHandle = null
}
}