Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* PROACTIVE remote-OAuth token refresh + DCR client-secret lifecycle sweep
* (mini-spec L). Renews the access token of eligible remote grants BEFORE they
* expire — its real value is exercising a rotating refresh token that an
* unattended (background / context-shared) grant would otherwise let lapse in
* silence — and observes DCR confidential clients whose `client_secret` is at or
* near expiry.
*
* The refresh machinery is NOT reimplemented: each candidate is fed to the
* existing reactive `getAccessToken` with `refreshBufferMs = Bp` and
* `requireBackground` set per grantKind (`true` for `user` grants — SEC-5;
* `false` for `shared` context identity, which is unattended by design), and the
* returned access token is DISCARDED — only the side effect (persisted renewed
* token) and the outcome label matter (mini-spec §6.2). Which grants are
* candidates is decided by the SQL `WHERE` of `listRemoteGrantsInProactiveWindow`
* plus `getAccessToken`'s own staleness check — NOT by a separate decision table.
* The pure `proactiveRefreshPolicy` module only maps each result to a metric
* outcome (`resultToOutcome`) and classifies DCR secret expiry
* (`classifyDcrSecretDecision`). This file is the thin orchestrator: enumerate →
* claim → refresh → label, plus the time-based DCR sweep.
*
* Best-effort, never worsens a grant: any per-row failure is logged + counted and
* the sweep continues; a transient refresh failure leaves the row untouched and
* degrades to the reactive path (mini-spec §4). The cron is gated OFF by default
* (`OAUTH_PROACTIVE_REFRESH_CRON_ENABLED`).
*/
import { config } from '../config.js'
import type { DbClient, DbTransactionClient } from '../db.js'
import { boundOAuthRefreshLockIdleTimeout, pool, withTransaction } from '../db.js'
import type { K8sGateway } from '../k8s.js'
import {
RecipeNotFoundError,
type RecipeWithOAuthClients,
SecretNotFoundError,
type SecretReader,
} from '../oauth/callback.js'
import {
type ExpiringDynamicClient,
listExpiringDynamicClients,
} from '../oauth/dynamicClientStore.js'
import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'
import { isSecretNotFound } from '../oauth/integrationNotConfigured.js'
import {
type ProactiveRefreshOutcome,
classifyDcrSecretDecision,
resultToOutcome,
} from '../oauth/proactiveRefreshPolicy.js'
import {
type OAuthGrantKey,
claimRemoteGrantForRefresh,
listRemoteGrantsInProactiveWindow,
} from '../oauth/store.js'
import { REACTIVE_REFRESH_BUFFER_MS, getAccessToken } from '../oauth/tokenHelper.js'
import { rootLogger } from '../observability/logger.js'
import {
oauthDcrSecretStatusTotal,
oauthProactiveRefreshDurationSeconds,
oauthProactiveRefreshGrantsTotal,
oauthProactiveRefreshRunsTotal,
} from '../observability/metrics.js'
import { type McpServerResource, normalizeMcpServerOwnerDecl } from '../routes/mcpOauth.js'
import { K8sNotFoundError } from './resourceService.js'
const log = rootLogger.child({ service: 'oauth-proactive-refresh' })
/** Buffers + windows for one sweep (all milliseconds). */
export interface ProactiveRefreshSweepOptions {
/** Bp — proactive buffer. */
proactiveBufferMs: number
/** Br — reactive buffer; the sweep never touches grants already inside it. */
reactiveBufferMs: number
/** Wc — DCR client-secret warn window. */
dcrWarnMs: number
}
/**
* Injectable IO for the sweep, so the orchestration (outcomes, per-server-CR DCR
* dedup) is unit-testable without a real Postgres or K8s. Production wiring is
* {@link defaultSweepDeps}.
*/
export interface ProactiveRefreshSweepDeps {
/** Snapshot reads (candidate enumeration + DCR list) — no lock, no txn. */
db: DbClient
/**
* Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). The client
* is transaction-scoped (branded {@link DbTransactionClient}) so the sweep can
* bound the carrier's idle timeout across the in-transaction refresh POST, same
* as the reactive path.
*/
runInTransaction: <T>(work: (txDb: DbTransactionClient) => Promise<T>) => Promise<T>
encryptionKey: Buffer
fetchFn: typeof fetch
}
/** Observable result of one sweep (assert this, not internal call counts — T4). */
export interface ProactiveRefreshSweepSummary {
candidates: number
outcomes: Record<ProactiveRefreshOutcome, number>
/** Deduped server-CR coordinates whose refresh failed `invalid_client` (§5). */
dcrClientInvalidServers: string[]
dcrExpiring: number
dcrExpired: number
}
function emptyOutcomes(): Record<ProactiveRefreshOutcome, number> {
return { ok: 0, transient: 0, client_invalid: 0, no_grant: 0, skipped: 0, error: 0 }
}
/** Stable per-server-CR coordinate for DCR-policy dedup (mini-spec §5 idempotency). */
function serverCoord(key: OAuthGrantKey): string {
return `mcpserver|${key.recipeNamespace}|${key.recipeName}`
}
/**
* Read an mcpserver CR and normalize it to the owner-agnostic oauthClients shape
* the refresh path consumes — IDENTICAL wiring to the broker route (D4, no
* drift). Not reused directly because the broker builds it inside its router;
* both call the same `normalizeMcpServerOwnerDecl`.
*/
function buildOwnerDeclReader(gateway: K8sGateway): {
read(name: string): Promise<RecipeWithOAuthClients | null>
} {
return {
async read(name): Promise<RecipeWithOAuthClients | null> {
try {
const server = (await gateway.getResource(
'mcpservers',
name,
config.mcpServersNamespace
)) as McpServerResource
return normalizeMcpServerOwnerDecl(server)
} catch (err) {
if (err instanceof K8sNotFoundError) {
throw new RecipeNotFoundError(`mcpserver ${config.mcpServersNamespace}/${name} not found`)
}
throw err
}
},
}
}
function buildSecretReader(gateway: K8sGateway): SecretReader {
return {
async read(name, namespace): Promise<Record<string, string>> {
try {
const raw = (await gateway.getSecret(name, namespace)) as { data?: Record<string, string> }
const decoded: Record<string, string> = {}
for (const [k, v] of Object.entries(raw.data ?? {})) {
decoded[k] = Buffer.from(v, 'base64').toString('utf8')
}
return decoded
} catch (err) {
if (isSecretNotFound(err)) {
throw new SecretNotFoundError(`secret ${namespace}/${name} not found`)
}
throw err
}
},
}
}
/** Production IO wiring: the core pool, the real encryption key, real `fetch`. */
export function defaultSweepDeps(): ProactiveRefreshSweepDeps {
return {
db: { query: (text, values) => pool.query(text, values) },
// `withTransaction` already owns the BEGIN/COMMIT/ROLLBACK/release dance
// (including the finicky release-on-error handling); the sweep passes the
// in-transaction client straight through as its `deps.db`, so the refresh
// UPDATE lands on the SAME connection that holds the `FOR UPDATE` lock.
runInTransaction: work => withTransaction(txDb => work(txDb)),
encryptionKey: deriveOAuthEncryptionKey(config.oauthEncryptionKey),
fetchFn: (input, init) => fetch(input, init),
}
}
/**
* One proactive-refresh + DCR sweep. Enumerates remote grants in the proactive
* window, claims each under `FOR UPDATE SKIP LOCKED` in a short transaction, and
* refreshes it via the reactive machinery; then sweeps DCR client-secret expiry.
* Best-effort: a per-row throw is counted as `error` and the sweep continues.
*/
export async function runProactiveRefreshSweep(
gateway: K8sGateway,
opts: ProactiveRefreshSweepOptions,
deps: ProactiveRefreshSweepDeps
): Promise<ProactiveRefreshSweepSummary> {
const window = {
proactiveBufferMs: opts.proactiveBufferMs,
reactiveBufferMs: opts.reactiveBufferMs,
}
const ownerDeclReader = buildOwnerDeclReader(gateway)
const secretReader = buildSecretReader(gateway)
const summary: ProactiveRefreshSweepSummary = {
candidates: 0,
outcomes: emptyOutcomes(),
dcrClientInvalidServers: [],
dcrExpiring: 0,
dcrExpired: 0,
}
const clientInvalidServers = new Set<string>()
const candidates = await listRemoteGrantsInProactiveWindow(deps.db, window)
summary.candidates = candidates.length
for (const key of candidates) {
let outcome: ProactiveRefreshOutcome
try {
outcome = await deps.runInTransaction(async txDb => {
// Re-claim under a row lock. 0 rows ⇒ another replica holds it, or the
// reactive path already renewed it out of the window since the snapshot.
const claimed = await claimRemoteGrantForRefresh(txDb, key, window)
if (!claimed) return 'skipped' as const
// The refresh POST runs INSIDE this transaction while it holds the row
// lock, so bound the idle-in-transaction tenancy just like the reactive
// path: `withTransaction` sets no such timeout, and if the AS hangs mid
// POST neither statement_timeout (no statement is running) nor the
// transport abort guarantees the lock + pool connection is released.
await boundOAuthRefreshLockIdleTimeout(txDb)
// requireBackground is PER-grantKind, not fixed. SEC-5 governs `user`
// grants only (unattended reuse needs explicit background consent), so
// those keep `true` — redundant with the enumeration filter, kept as
// defense in depth. `shared` grants are context identity, unattended by
// design and created by `bootstrapSharedOAuthGrant` with `background`
// defaulting to false; forcing `true` here would make `getOAuthGrant`'s
// shared branch add `AND background = true`, return `no_grant`, and never
// refresh the context grant — the mini-spec's headline case. Mirrors the
// interactive shared broker (`routes/mcpOauth.ts`, requireBackground:false).
// The access token is discarded; only the persisted refresh + the kind
// matter. refreshBufferMs = Bp so `getAccessToken` treats an
// in-proactive-window token as stale and refreshes it.
// resolveDns/pinnedTransport left undefined ⇒ production resolve +
// node:https (the remote lane's own IP-pinned path).
const result = await getAccessToken(
{ ...key, requireBackground: key.grantKind === 'user' },
{
db: txDb,
recipeReader: ownerDeclReader,
secretReader,
fetchFn: deps.fetchFn,
encryptionKey: deps.encryptionKey,
refreshBufferMs: opts.proactiveBufferMs,
}
)
return resultToOutcome(result)
})
} catch (err) {
// Best-effort: a single row must never abort the sweep. `withTransaction`
// already rolled back and released the connection.
outcome = 'error'
log.error(
{
event: 'oauth_proactive_refresh_row_error',
serverNamespace: key.recipeNamespace,
serverName: key.recipeName,
grantKind: key.grantKind,
oauthClientId: key.oauthClientId,
err: err instanceof Error ? err.message : String(err),
},
'proactive refresh row failed; skipping'
)
}
oauthProactiveRefreshGrantsTotal.inc({ outcome })
summary.outcomes[outcome] += 1
if (outcome === 'client_invalid') clientInvalidServers.add(serverCoord(key))
}
// §5 idempotency: N grants of one server-CR failing `invalid_client` apply the
// DCR policy ONCE per coordinate. Avisar/degradar only — never re-register.
summary.dcrClientInvalidServers = [...clientInvalidServers]
for (const coord of clientInvalidServers) {
log.warn(
{ event: 'oauth_dcr_client_invalid_runtime', server: coord },
'remote refresh failed invalid_client; DCR confidential secret likely expired/rotated — re-registration required (reactive path degrades to connect_required)'
)
}
// §5 time-based DCR sweep — metadata only, never decrypts. Independent of the
// runtime signal above and best-effort: a read failure logs but does not fail
// the run (the token refreshes above already committed).
try {
await sweepDcrSecretExpiry(deps.db, opts.dcrWarnMs, summary)
} catch (err) {
log.error(
{
event: 'oauth_dcr_secret_sweep_error',
err: err instanceof Error ? err.message : String(err),
},
'DCR secret-expiry sweep failed'
)
}
return summary
}
async function sweepDcrSecretExpiry(
db: DbClient,
dcrWarnMs: number,
summary: ProactiveRefreshSweepSummary
): Promise<void> {
const now = Date.now()
const rows: ExpiringDynamicClient[] = await listExpiringDynamicClients(db, {
withinMs: dcrWarnMs,
})
for (const row of rows) {
const decision = classifyDcrSecretDecision(
{ clientMode: row.clientMode, clientSecretExpiresAt: row.clientSecretExpiresAt },
now,
dcrWarnMs
)
if (decision.kind === 'warn') {
summary.dcrExpiring += 1
oauthDcrSecretStatusTotal.inc({ state: 'expiring' })
log.warn(
{
event: 'oauth_dcr_secret_expiring',
serverNamespace: row.serverNamespace,
serverName: row.serverName,
clientSecretExpiresAt: row.clientSecretExpiresAt.toISOString(),
},
'DCR confidential client_secret is nearing expiry (avisar; no auto re-registration in this unit)'
)
} else if (decision.kind === 'expired') {
summary.dcrExpired += 1
oauthDcrSecretStatusTotal.inc({ state: 'expired' })
log.warn(
{
event: 'oauth_dcr_secret_expired',
serverNamespace: row.serverNamespace,
serverName: row.serverName,
clientSecretExpiresAt: row.clientSecretExpiresAt.toISOString(),
},
'DCR confidential client_secret has expired; refresh will fail invalid_client and degrade to connect_required (re-registration required)'
)
}
// `noop` cannot occur given the enumeration filter, but is harmless if it does.
}
}
let intervalHandle: ReturnType<typeof setInterval> | null = null
/** Options for the cron loop; buffers/windows plus the tick interval. */
export interface OauthProactiveRefreshCronOptions extends ProactiveRefreshSweepOptions {
intervalMs: number
}
/**
* Start the proactive-refresh cron (idempotent; a second call is a no-op while
* one is running). Each tick runs {@link runProactiveRefreshSweep} and records
* `oauth_proactive_refresh_runs_total{result}` + the duration histogram. Never
* holds the process open (`unref`).
*/
export function startOauthProactiveRefreshCron(
gateway: K8sGateway,
opts: OauthProactiveRefreshCronOptions,
deps: ProactiveRefreshSweepDeps = defaultSweepDeps()
): void {
if (intervalHandle) return
intervalHandle = setInterval(() => {
void runOnce(gateway, opts, deps)
}, opts.intervalMs)
intervalHandle.unref()
log.info(
{
event: 'oauth_proactive_refresh_cron_started',
intervalMs: opts.intervalMs,
proactiveBufferMs: opts.proactiveBufferMs,
reactiveBufferMs: opts.reactiveBufferMs,
dcrWarnMs: opts.dcrWarnMs,
},
'oauth proactive refresh cron started'
)
}
async function runOnce(
gateway: K8sGateway,
opts: ProactiveRefreshSweepOptions,
deps: ProactiveRefreshSweepDeps
): Promise<void> {
const startHr = process.hrtime.bigint()
try {
const summary = await runProactiveRefreshSweep(gateway, opts, deps)
oauthProactiveRefreshRunsTotal.inc({ result: 'ok' })
log.debug(
{
event: 'oauth_proactive_refresh_sweep',
candidates: summary.candidates,
outcomes: summary.outcomes,
dcrExpiring: summary.dcrExpiring,
dcrExpired: summary.dcrExpired,
dcrClientInvalidServers: summary.dcrClientInvalidServers.length,
},
'oauth proactive refresh sweep complete'
)
} catch (err) {
oauthProactiveRefreshRunsTotal.inc({ result: 'error' })
log.error(
{
event: 'oauth_proactive_refresh_sweep_error',
err: err instanceof Error ? err.message : String(err),
},
'oauth proactive refresh sweep failed'
)
} finally {
const durationSec = Number(process.hrtime.bigint() - startHr) / 1e9
oauthProactiveRefreshDurationSeconds.observe(durationSec)
}
}
export function stopOauthProactiveRefreshCron(): void {
if (!intervalHandle) return
clearInterval(intervalHandle)
intervalHandle = null
}
/** Re-exported so `main.ts` passes the reactive buffer as the window's Br. */
export { REACTIVE_REFRESH_BUFFER_MS }