Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(oauth): proactive remote token refresh + DCR secret lifecycle sweep (fase L)
Diego Zacarias65a41088783e
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex f7a852a71..457f61f6a 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.597",+ "version": "0.1.598","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.597",+ "version": "0.1.598","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 82574c3bb..393630037 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.597",+ "version": "0.1.598","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex a953e2c73..b488d4fa8 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -1,5 +1,6 @@import { createPublicKey } from 'node:crypto'import { DEFAULT_ALLOWED_PLUGIN_IMAGE_PREFIXES } from '@clerum/image-policy'+import { REACTIVE_REFRESH_BUFFER_MS } from './oauth/tokenHelper.js'type Config = {port: number@@ -150,6 +151,18 @@ type Config = {// also guards implausibly-low provider counts.llmCatalogSyncProviderMinLive: numberregistryPullSecretReconcileIntervalMs: number+ // OAuth proactive-refresh cron (mini-spec L). Default OFF until e2e; renews+ // remote access tokens inside the proactive window so a background grant's+ // rotating refresh token is exercised before it lapses, and sweeps DCR+ // client-secret expiry (avisar/degradar only).+ oauthProactiveRefreshCronEnabled: boolean+ oauthProactiveRefreshIntervalMs: number+ // Bp — proactive buffer. A grant is a candidate while its access token expires+ // within this window. MUST be > REACTIVE_REFRESH_BUFFER_MS (Br); validated at+ // boot (mini-spec §3 invariant `Bp > Br`).+ oauthProactiveRefreshBufferMs: number+ // Wc — warn window for a DCR confidential client_secret nearing expiry.+ oauthDcrSecretWarnMs: numberbudgetReservationTtlSeconds: numberapprovalRetentionDays: numberuserApprovalRequestArchiveCronEnabled: boolean@@ -682,6 +695,23 @@ if ()}+// Bp (proactive buffer). Computed as a local so the `Bp > Br` invariant+// (mini-spec L §3) is validated BEFORE the config object is built — a proactive+// window that does not clear the reactive buffer would let the two paths compete+// for the same row on the happy path, defeating the idempotency argument. Br is+// the reactive buffer's single source of truth (`tokenHelper.ts`), never a+// re-typed literal. Fail-fast (throw) rather than clamp, like every other+// invalid config value here.+const oauthProactiveRefreshBufferMs = positiveIntegerFromEnv(+ 'OAUTH_PROACTIVE_REFRESH_BUFFER_MS',+ 300_000+)+if (oauthProactiveRefreshBufferMs <= REACTIVE_REFRESH_BUFFER_MS) {+ throw new Error(+ `OAUTH_PROACTIVE_REFRESH_BUFFER_MS (${oauthProactiveRefreshBufferMs}) must be greater than the reactive refresh buffer (${REACTIVE_REFRESH_BUFFER_MS}ms)`+ )+}+export const config: Config = {port: Number(process.env.CONTROL_API_PORT || 8090),jsonBodyLimit: process.env.CONTROL_API_JSON_BODY_LIMIT || '150mb',@@ -884,6 +914,18 @@ export const config: Config = {600_000,REGISTRY_PULL_SECRET_RECONCILE_MIN_INTERVAL_MS),+ // OAuth proactive-refresh cron (mini-spec L). Default OFF: `=== 'true'`, so the+ // sweep stays dark until an operator opts in after e2e.+ oauthProactiveRefreshCronEnabled: process.env.OAUTH_PROACTIVE_REFRESH_CRON_ENABLED === 'true',+ oauthProactiveRefreshIntervalMs: positiveIntegerFromEnv(+ 'OAUTH_PROACTIVE_REFRESH_INTERVAL_MS',+ 60_000+ ),+ // Computed + validated above the literal (Bp > Br must hold before the object+ // is built): see `oauthProactiveRefreshBufferMs` local.+ oauthProactiveRefreshBufferMs,+ // Wc — default 7 days.+ oauthDcrSecretWarnMs: positiveIntegerFromEnv('OAUTH_DCR_SECRET_WARN_MS', 604_800_000),// Danger-zone reservation TTL (§9.8a: ~2-3× the rollup lag, ~5 min). Short// enough that a hung reservation auto-frees; long enough that real spend has// reached the rollups before it expires (no double-count on the next check).diff --git a/control-api/src/main.ts b/control-api/src/main.tsindex c77b9d132..a929b5c72 100644--- a/control-api/src/main.ts+++ b/control-api/src/main.ts@@ -15,6 +15,11 @@ import {import { syncDiscoveredModels } from './services/llmCatalogSync.js'import { startLlmCatalogSyncCron, stopLlmCatalogSyncCron } from './services/llmCatalogSyncCron.js'import { runBootEnrollment } from './services/memberRegistrationEnrollment.js'+import {+ REACTIVE_REFRESH_BUFFER_MS,+ startOauthProactiveRefreshCron,+ stopOauthProactiveRefreshCron,+} from './services/oauthProactiveRefreshCron.js'import {startPluginWorkloadSdkMaintenanceCron,stopPluginWorkloadSdkMaintenanceCron,@@ -166,6 +171,22 @@ async function main(): Promise<void> {)}+ if (config.oauthProactiveRefreshCronEnabled) {+ startOauthProactiveRefreshCron(gateway, {+ intervalMs: config.oauthProactiveRefreshIntervalMs,+ proactiveBufferMs: config.oauthProactiveRefreshBufferMs,+ reactiveBufferMs: REACTIVE_REFRESH_BUFFER_MS,+ dcrWarnMs: config.oauthDcrSecretWarnMs,+ })+ console.log(+ `[ControlAPI] OAuth proactive refresh cron enabled (interval=${config.oauthProactiveRefreshIntervalMs}ms, Bp=${config.oauthProactiveRefreshBufferMs}ms, Wc=${config.oauthDcrSecretWarnMs}ms)`+ )+ } else {+ console.log(+ '[ControlAPI] OAuth proactive refresh cron disabled (OAUTH_PROACTIVE_REFRESH_CRON_ENABLED not "true")'+ )+ }+const server = new ControlApiServer(gateway, config.port)await server.start()@@ -196,6 +217,7 @@ main().catch(error => {stopUsageRetentionCron()stopBudgetReservationSweepCron()stopLlmCatalogSyncCron()+ stopOauthProactiveRefreshCron()stopWorkflowApprovalTraceProjector()void pool.end()process.exit(1)diff --git a/control-api/src/oauth/dynamicClientStore.ts b/control-api/src/oauth/dynamicClientStore.tsindex 8eb2f8f6b..9c072ac5a 100644--- a/control-api/src/oauth/dynamicClientStore.ts+++ b/control-api/src/oauth/dynamicClientStore.ts@@ -192,6 +192,58 @@ export async function getDynamicClient(}}+/**+ * Metadata-only view of a `dynamic_clients` row whose confidential secret is at+ * or near expiry — enough for the DCR lifecycle decision (§5) without touching+ * any encrypted material.+ */+export interface ExpiringDynamicClient {+ serverNamespace: string+ serverName: string+ clientMode: 'confidential'+ /** Non-null by construction (the enumeration filters out NULL expiry). */+ clientSecretExpiresAt: Date+}++/**+ * Enumerate confidential dynamic clients whose `client_secret_expires_at` is at+ * or before `now + withinMs` (mini-spec L §5). METADATA-ONLY: it never decrypts+ * the secret or the registration token, so it needs no `encryptionKey`.+ *+ * The filter already realizes the §5 table's structure: `public` and+ * non-expiring rows are excluded (C1/C2), healthy rows fall outside `withinMs`+ * (C3), and the returned set covers BOTH expiring (C4, `now < exp`) and expired+ * (C5, `exp ≤ now`) — the caller splits them with `classifyDcrSecretDecision`.+ */+export async function listExpiringDynamicClients(+ db: DbClient,+ opts: { withinMs: number }+): Promise<ExpiringDynamicClient[]> {+ const result = await db.query(+ `SELECT server_namespace, server_name, client_mode, client_secret_expires_at+ FROM dynamic_clients+ WHERE client_mode = 'confidential'+ AND client_secret_expires_at IS NOT NULL+ AND client_secret_expires_at <= NOW() + ($1::bigint * INTERVAL '1 millisecond')+ ORDER BY server_namespace, server_name`,+ [opts.withinMs]+ )+ return result.rows.map(r => {+ const row = r as {+ server_namespace: string+ server_name: string+ client_mode: 'confidential'+ client_secret_expires_at: Date+ }+ return {+ serverNamespace: row.server_namespace,+ serverName: row.server_name,+ clientMode: row.client_mode,+ clientSecretExpiresAt: row.client_secret_expires_at,+ }+ })+}+/*** Hard-delete the dynamic client for a server CR (idempotent). Returns the* number of rows removed so a caller can audit whether anything was revokeddiff --git a/control-api/src/oauth/proactiveRefreshPolicy.ts b/control-api/src/oauth/proactiveRefreshPolicy.tsnew file mode 100644index 000000000..d9dfe2eeb--- /dev/null+++ b/control-api/src/oauth/proactiveRefreshPolicy.ts@@ -0,0 +1,170 @@+import type { GetAccessTokenResult } from './tokenHelper.js'++/**+ * Pure decision layer for the PROACTIVE remote-token refresh + DCR client+ * lifecycle sweep (mini-spec L §4/§5). This module holds NO IO: every function+ * is `(input, now, buffers) → decision`, so the two decision tables are+ * property-testable in isolation (pr-discipline T2) and the cron+ * (`services/oauthProactiveRefreshCron.ts`) stays a thin orchestrator over them.+ *+ * The refresh machinery itself is NOT reimplemented here — the cron feeds each+ * `refresh` decision into the existing reactive `getAccessToken` (mini-spec §6.2).+ */++// ─── §4 · Proactive token-refresh decision ──────────────────────────────────++/**+ * The subset of an `oauth_grants` row the token decision depends on. Derived by+ * the caller from the enumerated row; `accessTokenExpiresAt` is the authoritative+ * expiry (NULL ⇒ the AS never supplied `expires_in`).+ */+export interface ProactiveTokenInput {+ /** Authoritative access-token expiry; NULL ⇒ no computable threshold (P1). */+ accessTokenExpiresAt: Date | null+ /** Whether a refresh token is stored for the grant. */+ hasRefreshToken: boolean+ /** `spec.oauth.supportsRefresh`, fail-closed from pinned metadata (D-8). */+ supportsRefresh: boolean+ /** Whether the grant is eligible for unattended (background) use (P7). */+ eligibleForBackground: boolean+}++/**+ * A noop carries WHY it was a noop so the cron can emit a precise metric/log+ * without re-deriving the reason. `reactive_window` = the grant is inside the+ * REACTIVE window (`remaining ≤ Br`); the reactive path owns it, not the cron.+ */+export type ProactiveTokenDecision =+ | {+ kind: 'noop'+ reason: 'not_eligible' | 'null_expiry' | 'expired' | 'healthy' | 'reactive_window'+ }+ | { kind: 'refresh' }+ | { kind: 'reconsent_imminent'; reason: 'no_refresh_support' | 'no_refresh_token' }++/**+ * Classify one candidate grant against the proactive window (mini-spec §4).+ *+ * Precedence (each branch is exclusive; the first match wins):+ * 1. `!eligibleForBackground` → noop `not_eligible` (P7; the "cualquiera" gate)+ * 2. `accessTokenExpiresAt == null`→ noop `null_expiry` (P1)+ * 3. `remaining ≤ 0` → noop `expired` (P6; proactive never chases dead tokens)+ * 4. `remaining > Bp` → noop `healthy` (P2; outside the window, healthy)+ * 5. `remaining ≤ Br` → noop `reactive_window` (below the proactive window)+ * 6. in-window + `!supportsRefresh`→ reconsent_imminent (P4; fail-closed, no refresh attempt)+ * 7. in-window + no refresh token → reconsent_imminent (P5)+ * 8. in-window, refreshable → refresh (P3)+ *+ * Proactive window is the strict-open/closed interval `(now+Br, now+Bp]` i.e.+ * `Br < remaining ≤ Bp`. `Bp > Br` is a config invariant (validated at boot).+ *+ * @param now epoch milliseconds+ * @param bpMs proactive buffer (Bp)+ * @param brMs reactive buffer (Br)+ */+export function classifyProactiveTokenDecision(+ input: ProactiveTokenInput,+ now: number,+ bpMs: number,+ brMs: number+): ProactiveTokenDecision {+ if (!input.eligibleForBackground) return { kind: 'noop', reason: 'not_eligible' }+ if (input.accessTokenExpiresAt === null) return { kind: 'noop', reason: 'null_expiry' }++ const remaining = input.accessTokenExpiresAt.getTime() - now+ if (remaining <= 0) return { kind: 'noop', reason: 'expired' }+ if (remaining > bpMs) return { kind: 'noop', reason: 'healthy' }+ if (remaining <= brMs) return { kind: 'noop', reason: 'reactive_window' }++ // Inside the proactive window (Br < remaining ≤ Bp).+ if (!input.supportsRefresh) return { kind: 'reconsent_imminent', reason: 'no_refresh_support' }+ if (!input.hasRefreshToken) return { kind: 'reconsent_imminent', reason: 'no_refresh_token' }+ return { kind: 'refresh' }+}++// ─── §4 · Refresh-result interpretation ─────────────────────────────────────++/**+ * Outcome label for `oauth_proactive_refresh_grants_total{outcome}`. `skipped`+ * is NOT produced by {@link resultToOutcome} — the cron emits it directly when a+ * `FOR UPDATE SKIP LOCKED` claim finds 0 rows (another replica holds it, or the+ * reactive path already renewed it out of the window).+ */+export type ProactiveRefreshOutcome =+ | 'ok'+ | 'transient'+ | 'client_invalid'+ | 'no_grant'+ | 'skipped'+ | 'error'++/**+ * Map a `getAccessToken` result to a proactive outcome (mini-spec §4, second+ * table). `refresh_failed` with a 400/401 status is a client-credential failure+ * (`invalid_client`) that triggers the DCR policy (§5); any other+ * `refresh_failed` is transient and degrades to the reactive path. Config-broken+ * kinds are `error` and are never retried in a tight loop.+ */+export function resultToOutcome(result: GetAccessTokenResult): ProactiveRefreshOutcome {+ switch (result.kind) {+ case 'ok':+ return 'ok'+ case 'no_grant':+ return 'no_grant'+ case 'refresh_failed':+ return result.status === 400 || result.status === 401 ? 'client_invalid' : 'transient'+ case 'recipe_not_found':+ case 'unknown_oauth_client':+ case 'unsupported_provider':+ case 'secret_missing':+ return 'error'+ }+}++// ─── §5 · DCR client-secret lifecycle decision ──────────────────────────────++/** The subset of a `dynamic_clients` row the DCR decision depends on (§5). */+export interface DcrSecretInput {+ clientMode: 'public' | 'confidential'+ /** RFC 7591 `client_secret_expires_at`; NULL ⇒ non-expiring (0/absent). */+ clientSecretExpiresAt: Date | null+}++/**+ * DCR lifecycle decision (mini-spec §5). `warn` = C4 (expiring within Wc);+ * `expired` = C5 (already lapsed → the confidential refresh will fail+ * `invalid_client` and the reactive path degrades to `connect_required`). Both+ * are avisar/degrade only — this unit never re-registers or mutates (C6 is+ * transparent and needs no branch here).+ */+export type DcrSecretDecision =+ | { kind: 'noop'; reason: 'public' | 'no_expiry' | 'healthy' }+ | { kind: 'warn' }+ | { kind: 'expired' }++/**+ * Classify one `dynamic_clients` row against the warn window `Wc` (mini-spec §5).+ *+ * Precedence:+ * 1. `public` → noop `public` (C1)+ * 2. `clientSecretExpiresAt == null` → noop `no_expiry`(C2)+ * 3. `exp ≤ now` → expired (C5)+ * 4. `now < exp ≤ now+Wc` → warn (C4)+ * 5. else (`exp > now+Wc`) → noop `healthy` (C3)+ *+ * @param now epoch milliseconds+ * @param wcMs warn window (Wc)+ */+export function classifyDcrSecretDecision(+ input: DcrSecretInput,+ now: number,+ wcMs: number+): DcrSecretDecision {+ if (input.clientMode === 'public') return { kind: 'noop', reason: 'public' }+ if (input.clientSecretExpiresAt === null) return { kind: 'noop', reason: 'no_expiry' }++ const exp = input.clientSecretExpiresAt.getTime()+ if (exp <= now) return { kind: 'expired' }+ if (exp <= now + wcMs) return { kind: 'warn' }+ return { kind: 'noop', reason: 'healthy' }+}diff --git a/control-api/src/oauth/store.ts b/control-api/src/oauth/store.tsindex 495710d04..d83c4762b 100644--- a/control-api/src/oauth/store.ts+++ b/control-api/src/oauth/store.ts@@ -678,3 +678,167 @@ export async function listBackgroundUserGrants()return result.rows.map(r => (r as { user_id: string }).user_id)}++/**+ * Buffers (milliseconds) that define the proactive window on the token's life:+ * a grant is a proactive-refresh candidate while+ * `now + reactiveBufferMs < access_token_expires_at ≤ now + proactiveBufferMs`.+ * `proactiveBufferMs` = Bp, `reactiveBufferMs` = Br (Bp > Br, config invariant).+ */+export interface ProactiveWindow {+ proactiveBufferMs: number+ reactiveBufferMs: number+}++interface RemoteGrantWindowDbRow {+ owner_kind: OAuthOwnerKind+ recipe_namespace: string+ recipe_name: string+ user_id: string | null+ context_id: string | null+ oauth_client_id: string+ grant_kind: 'user' | 'service' | 'shared'+}++/**+ * Reconstruct the flavored {@link OAuthGrantKey} for an mcpserver-owned remote+ * grant enumerated by {@link listRemoteGrantsInProactiveWindow}. The enumeration+ * filter admits only `shared` and background `user` grants, so those are the only+ * two flavors handled; a `service`/unexpected row throws (fail loud, never mint a+ * malformed key).+ */+function remoteGrantKeyFromRow(row: RemoteGrantWindowDbRow): OAuthGrantKey {+ if (row.grant_kind === 'user') {+ if (!row.user_id) throw new Error('remote user grant row missing user_id')+ return {+ grantKind: 'user',+ ownerKind: 'mcpserver',+ recipeNamespace: row.recipe_namespace,+ recipeName: row.recipe_name,+ userId: row.user_id,+ oauthClientId: row.oauth_client_id,+ }+ }+ if (row.grant_kind === 'shared') {+ if (!row.context_id) throw new Error('remote shared grant row missing context_id')+ return {+ grantKind: 'shared',+ ownerKind: 'mcpserver',+ recipeNamespace: row.recipe_namespace,+ recipeName: row.recipe_name,+ contextId: row.context_id,+ oauthClientId: row.oauth_client_id,+ }+ }+ throw new Error(`unexpected grant_kind for remote proactive candidate: ${row.grant_kind}`)+}++/**+ * Enumerate the mcpserver-owned REMOTE grants whose access token sits inside the+ * proactive window (mini-spec L §6.3). A snapshot SELECT with NO lock — the cron+ * re-claims each row under `FOR UPDATE SKIP LOCKED` in its own short transaction+ * ({@link claimRemoteGrantForRefresh}) so no pool connection is pinned across the+ * refresh POSTs.+ *+ * Filter (§4 eligibility + window): `provider='remote'`, non-null expiry in+ * `(now+Br, now+Bp]`, and `grant_kind='shared' OR (grant_kind='user' AND+ * background=true)` (SEC-5: unattended use only — non-background user grants are+ * excluded here as a first line of defense, `requireBackground:true` being the+ * second). Returns the flavored keys; tokens are never read.+ */+export async function listRemoteGrantsInProactiveWindow(+ db: DbClient,+ window: ProactiveWindow+): Promise<OAuthGrantKey[]> {+ const result = await db.query(+ `SELECT owner_kind, recipe_namespace, recipe_name, user_id, context_id,+ oauth_client_id, grant_kind+ FROM oauth_grants+ WHERE owner_kind = 'mcpserver'+ AND provider = 'remote'+ AND access_token_expires_at IS NOT NULL+ AND access_token_expires_at > NOW() + ($1::bigint * INTERVAL '1 millisecond')+ AND access_token_expires_at <= NOW() + ($2::bigint * INTERVAL '1 millisecond')+ AND (grant_kind = 'shared' OR (grant_kind = 'user' AND background = true))+ ORDER BY recipe_namespace, recipe_name, oauth_client_id, grant_kind,+ user_id NULLS FIRST, context_id NULLS FIRST`,+ [window.reactiveBufferMs, window.proactiveBufferMs]+ )+ return result.rows.map(r => remoteGrantKeyFromRow(r as RemoteGrantWindowDbRow))+}++/**+ * Claim one enumerated grant for proactive refresh inside an open transaction+ * (mini-spec L §6.3). Runs `SELECT 1 … FOR UPDATE SKIP LOCKED` keyed by the exact+ * grant coordinate AND re-checks the proactive window, so a row another replica+ * already holds, or one the reactive path renewed out of the window since the+ * snapshot, is skipped (returns `false`). On `true` the caller holds the row lock+ * for the rest of the transaction and MUST run the refresh UPDATE on the SAME+ * `txClient` so it lands under this lock.+ *+ * `txClient` MUST be a client inside an open transaction (BEGIN issued); the lock+ * only survives until COMMIT/ROLLBACK.+ */+export async function claimRemoteGrantForRefresh(+ txClient: DbClient,+ key: OAuthGrantKey,+ window: ProactiveWindow+): Promise<boolean> {+ const ownerKind = resolveOwnerKind(key)+ // The window buffers are the FIRST two params ($1 = reactive, $2 = proactive)+ // of every branch so the window predicate is byte-identical across flavors and+ // needs no string surgery. The flavor's key params follow from $3.+ const windowPredicate = `access_token_expires_at IS NOT NULL+ AND access_token_expires_at > NOW() + ($1::bigint * INTERVAL '1 millisecond')+ AND access_token_expires_at <= NOW() + ($2::bigint * INTERVAL '1 millisecond')`+ const windowParams = [window.reactiveBufferMs, window.proactiveBufferMs]++ if (key.grantKind === 'user') {+ const result = await txClient.query(+ `SELECT 1 FROM oauth_grants+ WHERE ${windowPredicate}+ AND owner_kind = $3 AND recipe_namespace = $4 AND recipe_name = $5+ AND user_id = $6 AND oauth_client_id = $7 AND grant_kind = 'user'+ FOR UPDATE SKIP LOCKED`,+ [+ ...windowParams,+ ownerKind,+ key.recipeNamespace,+ key.recipeName,+ key.userId,+ key.oauthClientId,+ ]+ )+ return (result.rowCount ?? 0) > 0+ }++ if (key.grantKind === 'shared') {+ const result = await txClient.query(+ `SELECT 1 FROM oauth_grants+ WHERE ${windowPredicate}+ AND owner_kind = $3 AND recipe_namespace = $4 AND recipe_name = $5+ AND user_id IS NULL AND context_id = $6 AND oauth_client_id = $7+ AND grant_kind = 'shared'+ FOR UPDATE SKIP LOCKED`,+ [+ ...windowParams,+ ownerKind,+ key.recipeNamespace,+ key.recipeName,+ key.contextId,+ key.oauthClientId,+ ]+ )+ return (result.rowCount ?? 0) > 0+ }++ const result = await txClient.query(+ `SELECT 1 FROM oauth_grants+ WHERE ${windowPredicate}+ AND owner_kind = $3 AND recipe_namespace = $4 AND recipe_name = $5+ AND user_id IS NULL AND oauth_client_id = $6 AND grant_kind = 'service'+ FOR UPDATE SKIP LOCKED`,+ [...windowParams, ownerKind, key.recipeNamespace, key.recipeName, key.oauthClientId]+ )+ return (result.rowCount ?? 0) > 0+}diff --git a/control-api/src/oauth/tokenHelper.ts b/control-api/src/oauth/tokenHelper.tsindex e8732a9a4..cede9ee31 100644--- a/control-api/src/oauth/tokenHelper.ts+++ b/control-api/src/oauth/tokenHelper.ts@@ -30,6 +30,15 @@ import { type OAuthGrantKey, getOAuthGrant, refreshOAuthGrantTokens } from './st* for `service` grants (Path B, spec §10).*/+/**+ * Reactive refresh buffer (Br): a stored access token is treated as stale, and+ * refreshed on demand, once it is within this window of expiry. The default when+ * a caller passes no `refreshBufferMs`. Exported so the proactive buffer (Bp) can+ * be validated `Bp > Br` at config load (mini-spec L §3) against the SAME source+ * of truth the reactive path uses — never a re-typed literal.+ */+export const REACTIVE_REFRESH_BUFFER_MS = 60_000+export type GetAccessTokenInput = OAuthGrantKey & {/*** Require a background-consented grant (per-user broker, SEC-5). Parametrized,@@ -75,7 +84,7 @@ export async function getAccessToken(const grant = await getOAuthGrant(deps.db, deps.encryptionKey, input)if (!grant) return { kind: 'no_grant' }- const refreshBufferMs = deps.refreshBufferMs ?? 60_000+ const refreshBufferMs = deps.refreshBufferMs ?? REACTIVE_REFRESH_BUFFER_MSconst stillValid =!grant.accessTokenExpiresAt ||grant.accessTokenExpiresAt.getTime() - refreshBufferMs > Date.now()diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 4ad66acfa..b219101bb 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -378,6 +378,33 @@ export const pluginWorkloadSdkMaintenanceRunsTotal = getOrCreateCounter({labelNames: ['result'] as const as Array<'result'>, // ok | error})+// ─── OAuth proactive-refresh cron metrics (mini-spec L) ──────────────────+export const oauthProactiveRefreshRunsTotal = getOrCreateCounter({+ name: 'oauth_proactive_refresh_runs_total',+ help: 'Count of OAuth proactive-refresh cron sweeps.',+ labelNames: ['result'] as const as Array<'result'>, // ok | error+})++export const oauthProactiveRefreshGrantsTotal = getOrCreateCounter({+ name: 'oauth_proactive_refresh_grants_total',+ help: 'Count of remote grants processed by the proactive-refresh cron, labelled by outcome.',+ // ok | transient | client_invalid | no_grant | skipped | error+ labelNames: ['outcome'] as const as Array<'outcome'>,+})++export const oauthDcrSecretStatusTotal = getOrCreateCounter({+ name: 'oauth_dcr_secret_status_total',+ help: 'Count of DCR confidential clients observed near/at client_secret expiry.',+ labelNames: ['state'] as const as Array<'state'>, // expiring | expired+})++export const oauthProactiveRefreshDurationSeconds = getOrCreateHistogram({+ name: 'oauth_proactive_refresh_duration_seconds',+ help: 'Duration of each OAuth proactive-refresh cron sweep.',+ labelNames: [] as string[],+ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],+})+// ─── Governed tracing foundation ──────────────────────────────────────────// Keep these dimensions bounded. Run, event, request, session, human, agent,// team, recipe, Host, workload, and correlation identifiers are trace data,diff --git a/control-api/src/services/oauthProactiveRefreshCron.ts b/control-api/src/services/oauthProactiveRefreshCron.tsnew file mode 100644index 000000000..f78a0cd61--- /dev/null+++ b/control-api/src/services/oauthProactiveRefreshCron.ts@@ -0,0 +1,393 @@+/**+ * PROACTIVE remote-OAuth token refresh + DCR client-secret lifecycle sweep+ * (mini-spec L). Renews the access token of eligible remote grants BEFORE they+ * expire — its real value is exercising a rotating refresh token that an+ * unattended (background / context-shared) grant would otherwise let lapse in+ * silence — and observes DCR confidential clients whose `client_secret` is at or+ * near expiry.+ *+ * The refresh machinery is NOT reimplemented: each candidate is fed to the+ * existing reactive `getAccessToken` with `refreshBufferMs = Bp` and+ * `requireBackground: true`, and the returned access token is DISCARDED — only+ * the side effect (persisted renewed token) and the outcome label matter+ * (mini-spec §6.2). The decision layer is the pure `proactiveRefreshPolicy`+ * module; this file is the thin orchestrator: enumerate → claim → refresh →+ * classify, plus the time-based DCR sweep.+ *+ * Best-effort, never worsens a grant: any per-row failure is logged + counted and+ * the sweep continues; a transient refresh failure leaves the row untouched and+ * degrades to the reactive path (mini-spec §4). The cron is gated OFF by default+ * (`OAUTH_PROACTIVE_REFRESH_CRON_ENABLED`).+ */+import { config } from '../config.js'+import type { DbClient } from '../db.js'+import { pool, withTransaction } from '../db.js'+import type { K8sGateway } from '../k8s.js'+import {+ RecipeNotFoundError,+ type RecipeWithOAuthClients,+ SecretNotFoundError,+ type SecretReader,+} from '../oauth/callback.js'+import {+ type ExpiringDynamicClient,+ listExpiringDynamicClients,+} from '../oauth/dynamicClientStore.js'+import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'+import { isSecretNotFound } from '../oauth/integrationNotConfigured.js'+import {+ type ProactiveRefreshOutcome,+ classifyDcrSecretDecision,+ resultToOutcome,+} from '../oauth/proactiveRefreshPolicy.js'+import {+ type OAuthGrantKey,+ claimRemoteGrantForRefresh,+ listRemoteGrantsInProactiveWindow,+} from '../oauth/store.js'+import { REACTIVE_REFRESH_BUFFER_MS, getAccessToken } from '../oauth/tokenHelper.js'+import { rootLogger } from '../observability/logger.js'+import {+ oauthDcrSecretStatusTotal,+ oauthProactiveRefreshDurationSeconds,+ oauthProactiveRefreshGrantsTotal,+ oauthProactiveRefreshRunsTotal,+} from '../observability/metrics.js'+import { type McpServerResource, normalizeMcpServerOwnerDecl } from '../routes/mcpOauth.js'+import { K8sNotFoundError } from './resourceService.js'++const log = rootLogger.child({ service: 'oauth-proactive-refresh' })++/** Buffers + windows for one sweep (all milliseconds). */+export interface ProactiveRefreshSweepOptions {+ /** Bp — proactive buffer. */+ proactiveBufferMs: number+ /** Br — reactive buffer; the sweep never touches grants already inside it. */+ reactiveBufferMs: number+ /** Wc — DCR client-secret warn window. */+ dcrWarnMs: number+}++/**+ * Injectable IO for the sweep, so the orchestration (outcomes, per-server-CR DCR+ * dedup) is unit-testable without a real Postgres or K8s. Production wiring is+ * {@link defaultSweepDeps}.+ */+export interface ProactiveRefreshSweepDeps {+ /** Snapshot reads (candidate enumeration + DCR list) — no lock, no txn. */+ db: DbClient+ /** Runs `work` inside one short transaction (BEGIN…COMMIT/ROLLBACK). */+ runInTransaction: <T>(work: (txDb: DbClient) => Promise<T>) => Promise<T>+ encryptionKey: Buffer+ fetchFn: typeof fetch+}++/** Observable result of one sweep (assert this, not internal call counts — T4). */+export interface ProactiveRefreshSweepSummary {+ candidates: number+ outcomes: Record<ProactiveRefreshOutcome, number>+ /** Deduped server-CR coordinates whose refresh failed `invalid_client` (§5). */+ dcrClientInvalidServers: string[]+ dcrExpiring: number+ dcrExpired: number+}++function emptyOutcomes(): Record<ProactiveRefreshOutcome, number> {+ return { ok: 0, transient: 0, client_invalid: 0, no_grant: 0, skipped: 0, error: 0 }+}++/** Stable per-server-CR coordinate for DCR-policy dedup (mini-spec §5 idempotency). */+function serverCoord(key: OAuthGrantKey): string {+ return `mcpserver|${key.recipeNamespace}|${key.recipeName}`+}++/**+ * Read an mcpserver CR and normalize it to the owner-agnostic oauthClients shape+ * the refresh path consumes — IDENTICAL wiring to the broker route (D4, no+ * drift). Not reused directly because the broker builds it inside its router;+ * both call the same `normalizeMcpServerOwnerDecl`.+ */+function buildOwnerDeclReader(gateway: K8sGateway): {+ read(name: string): Promise<RecipeWithOAuthClients | null>+} {+ return {+ async read(name): Promise<RecipeWithOAuthClients | null> {+ try {+ const server = (await gateway.getResource(+ 'mcpservers',+ name,+ config.mcpServersNamespace+ )) as McpServerResource+ return normalizeMcpServerOwnerDecl(server)+ } catch (err) {+ if (err instanceof K8sNotFoundError) {+ throw new RecipeNotFoundError(`mcpserver ${config.mcpServersNamespace}/${name} not found`)+ }+ throw err+ }+ },+ }+}++function buildSecretReader(gateway: K8sGateway): SecretReader {+ return {+ async read(name, namespace): Promise<Record<string, string>> {+ try {+ const raw = (await gateway.getSecret(name, namespace)) as { data?: Record<string, string> }+ const decoded: Record<string, string> = {}+ for (const [k, v] of Object.entries(raw.data ?? {})) {+ decoded[k] = Buffer.from(v, 'base64').toString('utf8')+ }+ return decoded+ } catch (err) {+ if (isSecretNotFound(err)) {+ throw new SecretNotFoundError(`secret ${namespace}/${name} not found`)+ }+ throw err+ }+ },+ }+}++/** Production IO wiring: the core pool, the real encryption key, real `fetch`. */+export function defaultSweepDeps(): ProactiveRefreshSweepDeps {+ return {+ db: { query: (text, values) => pool.query(text, values) },+ // `withTransaction` already owns the BEGIN/COMMIT/ROLLBACK/release dance+ // (including the finicky release-on-error handling); the sweep passes the+ // in-transaction client straight through as its `deps.db`, so the refresh+ // UPDATE lands on the SAME connection that holds the `FOR UPDATE` lock.+ runInTransaction: work => withTransaction(txDb => work(txDb)),+ encryptionKey: deriveOAuthEncryptionKey(config.oauthEncryptionKey),+ fetchFn: (input, init) => fetch(input, init),+ }+}++/**+ * One proactive-refresh + DCR sweep. Enumerates remote grants in the proactive+ * window, claims each under `FOR UPDATE SKIP LOCKED` in a short transaction, and+ * refreshes it via the reactive machinery; then sweeps DCR client-secret expiry.+ * Best-effort: a per-row throw is counted as `error` and the sweep continues.+ */+export async function runProactiveRefreshSweep(+ gateway: K8sGateway,+ opts: ProactiveRefreshSweepOptions,+ deps: ProactiveRefreshSweepDeps+): Promise<ProactiveRefreshSweepSummary> {+ const window = {+ proactiveBufferMs: opts.proactiveBufferMs,+ reactiveBufferMs: opts.reactiveBufferMs,+ }+ const ownerDeclReader = buildOwnerDeclReader(gateway)+ const secretReader = buildSecretReader(gateway)++ const summary: ProactiveRefreshSweepSummary = {+ candidates: 0,+ outcomes: emptyOutcomes(),+ dcrClientInvalidServers: [],+ dcrExpiring: 0,+ dcrExpired: 0,+ }+ const clientInvalidServers = new Set<string>()++ const candidates = await listRemoteGrantsInProactiveWindow(deps.db, window)+ summary.candidates = candidates.length++ for (const key of candidates) {+ let outcome: ProactiveRefreshOutcome+ try {+ outcome = await deps.runInTransaction(async txDb => {+ // Re-claim under a row lock. 0 rows ⇒ another replica holds it, or the+ // reactive path already renewed it out of the window since the snapshot.+ const claimed = await claimRemoteGrantForRefresh(txDb, key, window)+ if (!claimed) return 'skipped' as const+ // requireBackground:true — unattended use (SEC-5). The access token is+ // discarded; only the persisted refresh + the kind matter. refreshBufferMs+ // = Bp so `getAccessToken` treats an in-proactive-window token as stale+ // and refreshes it. resolveDns/pinnedTransport left undefined ⇒ production+ // resolve + node:https (the remote lane's own IP-pinned path).+ const result = await getAccessToken(+ { ...key, requireBackground: true },+ {+ db: txDb,+ recipeReader: ownerDeclReader,+ secretReader,+ fetchFn: deps.fetchFn,+ encryptionKey: deps.encryptionKey,+ refreshBufferMs: opts.proactiveBufferMs,+ }+ )+ return resultToOutcome(result)+ })+ } catch (err) {+ // Best-effort: a single row must never abort the sweep. `withTransaction`+ // already rolled back and released the connection.+ outcome = 'error'+ log.error(+ {+ event: 'oauth_proactive_refresh_row_error',+ serverNamespace: key.recipeNamespace,+ serverName: key.recipeName,+ grantKind: key.grantKind,+ oauthClientId: key.oauthClientId,+ err: err instanceof Error ? err.message : String(err),+ },+ 'proactive refresh row failed; skipping'+ )+ }++ oauthProactiveRefreshGrantsTotal.inc({ outcome })+ summary.outcomes[outcome] += 1+ if (outcome === 'client_invalid') clientInvalidServers.add(serverCoord(key))+ }++ // §5 idempotency: N grants of one server-CR failing `invalid_client` apply the+ // DCR policy ONCE per coordinate. Avisar/degradar only — never re-register.+ summary.dcrClientInvalidServers = [...clientInvalidServers]+ for (const coord of clientInvalidServers) {+ log.warn(+ { event: 'oauth_dcr_client_invalid_runtime', server: coord },+ 'remote refresh failed invalid_client; DCR confidential secret likely expired/rotated — re-registration required (reactive path degrades to connect_required)'+ )+ }++ // §5 time-based DCR sweep — metadata only, never decrypts. Independent of the+ // runtime signal above and best-effort: a read failure logs but does not fail+ // the run (the token refreshes above already committed).+ try {+ await sweepDcrSecretExpiry(deps.db, opts.dcrWarnMs, summary)+ } catch (err) {+ log.error(+ {+ event: 'oauth_dcr_secret_sweep_error',+ err: err instanceof Error ? err.message : String(err),+ },+ 'DCR secret-expiry sweep failed'+ )+ }++ return summary+}++async function sweepDcrSecretExpiry(+ db: DbClient,+ dcrWarnMs: number,+ summary: ProactiveRefreshSweepSummary+): Promise<void> {+ const now = Date.now()+ const rows: ExpiringDynamicClient[] = await listExpiringDynamicClients(db, {+ withinMs: dcrWarnMs,+ })+ for (const row of rows) {+ const decision = classifyDcrSecretDecision(+ { clientMode: row.clientMode, clientSecretExpiresAt: row.clientSecretExpiresAt },+ now,+ dcrWarnMs+ )+ if (decision.kind === 'warn') {+ summary.dcrExpiring += 1+ oauthDcrSecretStatusTotal.inc({ state: 'expiring' })+ log.warn(+ {+ event: 'oauth_dcr_secret_expiring',+ serverNamespace: row.serverNamespace,+ serverName: row.serverName,+ clientSecretExpiresAt: row.clientSecretExpiresAt.toISOString(),+ },+ 'DCR confidential client_secret is nearing expiry (avisar; no auto re-registration in this unit)'+ )+ } else if (decision.kind === 'expired') {+ summary.dcrExpired += 1+ oauthDcrSecretStatusTotal.inc({ state: 'expired' })+ log.warn(+ {+ event: 'oauth_dcr_secret_expired',+ serverNamespace: row.serverNamespace,+ serverName: row.serverName,+ clientSecretExpiresAt: row.clientSecretExpiresAt.toISOString(),+ },+ 'DCR confidential client_secret has expired; refresh will fail invalid_client and degrade to connect_required (re-registration required)'+ )+ }+ // `noop` cannot occur given the enumeration filter, but is harmless if it does.+ }+}++let intervalHandle: ReturnType<typeof setInterval> | null = null++/** Options for the cron loop; buffers/windows plus the tick interval. */+export interface OauthProactiveRefreshCronOptions extends ProactiveRefreshSweepOptions {+ intervalMs: number+}++/**+ * Start the proactive-refresh cron (idempotent; a second call is a no-op while+ * one is running). Each tick runs {@link runProactiveRefreshSweep} and records+ * `oauth_proactive_refresh_runs_total{result}` + the duration histogram. Never+ * holds the process open (`unref`).+ */+export function startOauthProactiveRefreshCron(+ gateway: K8sGateway,+ opts: OauthProactiveRefreshCronOptions,+ deps: ProactiveRefreshSweepDeps = defaultSweepDeps()+): void {+ if (intervalHandle) return+ intervalHandle = setInterval(() => {+ void runOnce(gateway, opts, deps)+ }, opts.intervalMs)+ intervalHandle.unref()+ log.info(+ {+ event: 'oauth_proactive_refresh_cron_started',+ intervalMs: opts.intervalMs,+ proactiveBufferMs: opts.proactiveBufferMs,+ reactiveBufferMs: opts.reactiveBufferMs,+ dcrWarnMs: opts.dcrWarnMs,+ },+ 'oauth proactive refresh cron started'+ )+}++async function runOnce(+ gateway: K8sGateway,+ opts: ProactiveRefreshSweepOptions,+ deps: ProactiveRefreshSweepDeps+): Promise<void> {+ const startHr = process.hrtime.bigint()+ try {+ const summary = await runProactiveRefreshSweep(gateway, opts, deps)+ oauthProactiveRefreshRunsTotal.inc({ result: 'ok' })+ log.debug(+ {+ event: 'oauth_proactive_refresh_sweep',+ candidates: summary.candidates,+ outcomes: summary.outcomes,+ dcrExpiring: summary.dcrExpiring,+ dcrExpired: summary.dcrExpired,+ dcrClientInvalidServers: summary.dcrClientInvalidServers.length,+ },+ 'oauth proactive refresh sweep complete'+ )+ } catch (err) {+ oauthProactiveRefreshRunsTotal.inc({ result: 'error' })+ log.error(+ {+ event: 'oauth_proactive_refresh_sweep_error',+ err: err instanceof Error ? err.message : String(err),+ },+ 'oauth proactive refresh sweep failed'+ )+ } finally {+ const durationSec = Number(process.hrtime.bigint() - startHr) / 1e9+ oauthProactiveRefreshDurationSeconds.observe(durationSec)+ }+}++export function stopOauthProactiveRefreshCron(): void {+ if (!intervalHandle) return+ clearInterval(intervalHandle)+ intervalHandle = null+}++/** Re-exported so `main.ts` passes the reactive buffer as the window's Br. */+export { REACTIVE_REFRESH_BUFFER_MS }diff --git a/control-api/test/config.oauthProactiveRefresh.test.ts b/control-api/test/config.oauthProactiveRefresh.test.tsnew file mode 100644index 000000000..a119908bd--- /dev/null+++ b/control-api/test/config.oauthProactiveRefresh.test.ts@@ -0,0 +1,81 @@+/**+ * OAuth proactive-refresh cron config (mini-spec L §3/§5.1).+ *+ * The load-bearing invariant is `Bp > Br` (T5a): a proactive buffer that does not+ * clear the reactive buffer would let the proactive and reactive paths compete+ * for the same row on the happy path, defeating the idempotency argument. It is+ * refused at boot (throw), against the reactive buffer's single source of truth,+ * never a re-typed literal.+ */+import { afterEach, describe, expect, it, vi } from 'vitest'+import { REACTIVE_REFRESH_BUFFER_MS } from '../src/oauth/tokenHelper.js'++const BUFFER = 'OAUTH_PROACTIVE_REFRESH_BUFFER_MS'+const INTERVAL = 'OAUTH_PROACTIVE_REFRESH_INTERVAL_MS'+const ENABLED = 'OAUTH_PROACTIVE_REFRESH_CRON_ENABLED'+const WARN = 'OAUTH_DCR_SECRET_WARN_MS'+const KEYS = [BUFFER, INTERVAL, ENABLED, WARN] as const++async function loadConfigWith(env: Partial<Record<(typeof KEYS)[number], string>>) {+ const original = Object.fromEntries(KEYS.map(k => [k, process.env[k]]))+ for (const key of KEYS) delete process.env[key]+ for (const [key, value] of Object.entries(env)) process.env[key] = value+ vi.resetModules()+ try {+ const mod = await import('../src/config.js')+ return mod.config+ } finally {+ for (const key of KEYS) {+ const value = original[key]+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ }+}++describe('control-api oauth proactive-refresh config', () => {+ afterEach(() => {+ vi.resetModules()+ })++ it('defaults: cron off, 60s interval, Bp=300s, Wc=7d', async () => {+ const config = await loadConfigWith({})+ expect(config.oauthProactiveRefreshCronEnabled).toBe(false)+ expect(config.oauthProactiveRefreshIntervalMs).toBe(60_000)+ expect(config.oauthProactiveRefreshBufferMs).toBe(300_000)+ expect(config.oauthDcrSecretWarnMs).toBe(604_800_000)+ })++ it('enables only on the exact token "true"', async () => {+ expect((await loadConfigWith({ [ENABLED]: 'true' })).oauthProactiveRefreshCronEnabled).toBe(+ true+ )+ expect((await loadConfigWith({ [ENABLED]: 'TRUE' })).oauthProactiveRefreshCronEnabled).toBe(+ false+ )+ expect((await loadConfigWith({ [ENABLED]: '1' })).oauthProactiveRefreshCronEnabled).toBe(false)+ })++ it('accepts a Bp strictly greater than the reactive buffer Br', async () => {+ const config = await loadConfigWith({ [BUFFER]: String(REACTIVE_REFRESH_BUFFER_MS + 1) })+ expect(config.oauthProactiveRefreshBufferMs).toBe(REACTIVE_REFRESH_BUFFER_MS + 1)+ })++ it('rejects Bp equal to Br (T5a boundary)', async () => {+ await expect(loadConfigWith({ [BUFFER]: String(REACTIVE_REFRESH_BUFFER_MS) })).rejects.toThrow(+ /must be greater than the reactive refresh buffer/+ )+ })++ it('rejects Bp below Br', async () => {+ await expect(+ loadConfigWith({ [BUFFER]: String(REACTIVE_REFRESH_BUFFER_MS - 1) })+ ).rejects.toThrow(/must be greater than the reactive refresh buffer/)+ })++ it('accepts an interval and warn-window override', async () => {+ const config = await loadConfigWith({ [INTERVAL]: '120000', [WARN]: '1000' })+ expect(config.oauthProactiveRefreshIntervalMs).toBe(120_000)+ expect(config.oauthDcrSecretWarnMs).toBe(1_000)+ })+})diff --git a/control-api/test/oauth.proactiveRefresh.realPostgres.integration.test.ts b/control-api/test/oauth.proactiveRefresh.realPostgres.integration.test.tsnew file mode 100644index 000000000..e0d539bf0--- /dev/null+++ b/control-api/test/oauth.proactiveRefresh.realPostgres.integration.test.ts@@ -0,0 +1,273 @@+/**+ * Enumeration + claim for the proactive refresh sweep (mini-spec L §6.3), on a+ * REAL Postgres so `FOR UPDATE SKIP LOCKED`, the window predicate, and the+ * eligibility filter are exercised against the actual schema/migrations — not a+ * hand-built row shape. Fixtures come from the REAL producers+ * (`bootstrapSharedOAuthGrant` / `upsertOAuthGrant` / `upsertDynamicClient`, T1).+ *+ * CONTROL_API_REAL_PG_ADMIN_URL=postgres://<user>:<pass>@<host>:5432 npm test -- \+ * test/oauth.proactiveRefresh.realPostgres.integration.test.ts+ */+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import { config } from '../src/config.js'+import type { DbClient } from '../src/db.js'+import { initDb } from '../src/db.js'+import { listExpiringDynamicClients, upsertDynamicClient } from '../src/oauth/dynamicClientStore.js'+import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'+import {+ type OAuthGrantKey,+ bootstrapSharedOAuthGrant,+ claimRemoteGrantForRefresh,+ listRemoteGrantsInProactiveWindow,+ upsertOAuthGrant,+} from '../src/oauth/store.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++const KEY = deriveOAuthEncryptionKey(config.oauthEncryptionKey)+const NS = config.mcpServersNamespace+const WINDOW = { proactiveBufferMs: 300_000, reactiveBufferMs: 60_000 }+// Seconds-to-expiry that land a grant in each region of the token life.+const IN_WINDOW_SEC = 200 // 60s < 200s ≤ 300s → proactive window+const HEALTHY_SEC = 1_000 // > Bp+const REACTIVE_SEC = 30 // ≤ Br++describeRealPostgres('oauth proactive refresh — enumeration + claim (real Postgres)', () => {+ const database = `control_api_proactive_${randomUUID().replace(/-/g, '')}`+ let adminPool: Pool+ let dbPool: Pool+ let db: DbClient++ beforeAll(async () => {+ if (!adminUrl) throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required')+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)+ dbPool = new Pool({ connectionString: databaseUrl(adminUrl, database) })+ await initDb({ connect: () => dbPool.connect() })+ db = { query: (text, values) => dbPool.query(text, values) }+ })++ afterAll(async () => {+ await dbPool?.end()+ if (adminPool) {+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)+ await adminPool.end()+ }+ })++ const sharedKey = (server: string, ctx: string): OAuthGrantKey => ({+ grantKind: 'shared',+ ownerKind: 'mcpserver',+ recipeNamespace: NS,+ recipeName: server,+ contextId: ctx,+ oauthClientId: 'remote-cid',+ })++ async function seedShared(+ server: string,+ ctx: string,+ provider: string,+ expiresInSec: number | undefined+ ): Promise<void> {+ await bootstrapSharedOAuthGrant(db, KEY, {+ ...sharedKey(server, ctx),+ bootstrappedByUserId: 'user-1',+ provider,+ accessToken: 'AT',+ refreshToken: 'RT',+ accessTokenExpiresInSec: expiresInSec,+ })+ }++ async function seedUser(+ server: string,+ userId: string,+ background: boolean,+ expiresInSec: number+ ): Promise<void> {+ await upsertOAuthGrant(db, KEY, {+ grantKind: 'user',+ ownerKind: 'mcpserver',+ recipeNamespace: NS,+ recipeName: server,+ userId,+ oauthClientId: 'remote-cid',+ provider: 'remote',+ accessToken: 'AT',+ refreshToken: 'RT',+ accessTokenExpiresInSec: expiresInSec,+ })+ // The background column is set elsewhere at consent time; drive it directly+ // here (real schema column) to build the background-vs-not distinction.+ if (background) {+ await dbPool.query(+ `UPDATE oauth_grants SET background = true+ WHERE owner_kind = 'mcpserver' AND recipe_namespace = $1 AND recipe_name = $2+ AND user_id = $3 AND grant_kind = 'user'`,+ [NS, server, userId]+ )+ }+ }++ it('enumerates only remote mcpserver grants (shared + background user) inside the window', async () => {+ await seedShared('e-shared-in', 'ctx-in', 'remote', IN_WINDOW_SEC)+ await seedShared('e-shared-healthy', 'ctx-h', 'remote', HEALTHY_SEC)+ await seedShared('e-shared-reactive', 'ctx-r', 'remote', REACTIVE_SEC)+ await seedShared('e-shared-nullexp', 'ctx-n', 'remote', undefined)+ await seedShared('e-shared-baked', 'ctx-b', 'google', IN_WINDOW_SEC) // non-remote+ await seedUser('e-user-bg', 'u-bg', true, IN_WINDOW_SEC)+ await seedUser('e-user-nobg', 'u-nobg', false, IN_WINDOW_SEC)+ // Recipe-domain grant in-window: must never be touched by the mcpserver sweep.+ await upsertOAuthGrant(db, KEY, {+ grantKind: 'user',+ ownerKind: 'recipe',+ recipeNamespace: NS,+ recipeName: 'e-recipe',+ userId: 'u-recipe',+ oauthClientId: 'remote-cid',+ provider: 'remote',+ accessToken: 'AT',+ accessTokenExpiresInSec: IN_WINDOW_SEC,+ })++ const keys = await listRemoteGrantsInProactiveWindow(db, WINDOW)+ const names = keys.map(k => k.recipeName).sort()+ expect(names).toEqual(['e-shared-in', 'e-user-bg'])+ })++ it('FOR UPDATE SKIP LOCKED serializes the claim across connections', async () => {+ await seedShared('claim-a', 'ctx-1', 'remote', IN_WINDOW_SEC)+ const key = sharedKey('claim-a', 'ctx-1')++ const c1 = await dbPool.connect()+ const c2 = await dbPool.connect()+ try {+ await c1.query('BEGIN')+ await c2.query('BEGIN')+ const first = await claimRemoteGrantForRefresh(+ { query: (t, v) => c1.query(t, v) },+ key,+ WINDOW+ )+ const second = await claimRemoteGrantForRefresh(+ { query: (t, v) => c2.query(t, v) },+ key,+ WINDOW+ )+ expect(first).toBe(true)+ expect(second).toBe(false) // c1 holds the row lock; c2 skips it+ await c1.query('COMMIT')+ await c2.query('ROLLBACK')+ } finally {+ c1.release()+ c2.release()+ }+ })++ it('the claim re-checks the window: a grant renewed out of the window is not claimed', async () => {+ await seedShared('claim-b', 'ctx-1', 'remote', IN_WINDOW_SEC)+ const key = sharedKey('claim-b', 'ctx-1')+ // Simulate the reactive path having already renewed it to a full life.+ await dbPool.query(+ `UPDATE oauth_grants SET access_token_expires_at = NOW() + INTERVAL '1000 seconds'+ WHERE owner_kind = 'mcpserver' AND recipe_namespace = $1 AND recipe_name = $2+ AND context_id = $3 AND grant_kind = 'shared'`,+ [NS, 'claim-b', 'ctx-1']+ )+ const c1 = await dbPool.connect()+ try {+ await c1.query('BEGIN')+ const claimed = await claimRemoteGrantForRefresh(+ { query: (t, v) => c1.query(t, v) },+ key,+ WINDOW+ )+ expect(claimed).toBe(false)+ await c1.query('ROLLBACK')+ } finally {+ c1.release()+ }+ })++ it('T5(b): a successful proactive refresh moves the row out of the enumeration window', async () => {+ await seedShared('t5b', 'ctx-1', 'remote', IN_WINDOW_SEC)+ const before = await listRemoteGrantsInProactiveWindow(db, WINDOW)+ expect(before.some(k => k.recipeName === 't5b')).toBe(true)+ // A refresh pushes expiry to a full life ahead.+ await dbPool.query(+ `UPDATE oauth_grants SET access_token_expires_at = NOW() + INTERVAL '3600 seconds'+ WHERE owner_kind = 'mcpserver' AND recipe_namespace = $1 AND recipe_name = $2+ AND context_id = $3 AND grant_kind = 'shared'`,+ [NS, 't5b', 'ctx-1']+ )+ const after = await listRemoteGrantsInProactiveWindow(db, WINDOW)+ expect(after.some(k => k.recipeName === 't5b')).toBe(false)+ })++ it('listExpiringDynamicClients returns expiring + expired confidential clients only', async () => {+ const nowSec = Math.floor(Date.now() / 1000)+ await upsertDynamicClient(db, KEY, {+ serverNamespace: NS,+ serverName: 'dc-expiring',+ issuer: 'https://as.example.com',+ clientId: 'cid-1',+ clientMode: 'confidential',+ clientSecret: 'sec',+ clientSecretExpiresAtSec: nowSec + 3600, // within 7d+ })+ await upsertDynamicClient(db, KEY, {+ serverNamespace: NS,+ serverName: 'dc-expired',+ issuer: 'https://as.example.com',+ clientId: 'cid-2',+ clientMode: 'confidential',+ clientSecret: 'sec',+ clientSecretExpiresAtSec: nowSec - 3600, // already lapsed+ })+ await upsertDynamicClient(db, KEY, {+ serverNamespace: NS,+ serverName: 'dc-healthy',+ issuer: 'https://as.example.com',+ clientId: 'cid-3',+ clientMode: 'confidential',+ clientSecret: 'sec',+ clientSecretExpiresAtSec: nowSec + 400 * 24 * 3600, // far beyond Wc+ })+ await upsertDynamicClient(db, KEY, {+ serverNamespace: NS,+ serverName: 'dc-nonexpiring',+ issuer: 'https://as.example.com',+ clientId: 'cid-4',+ clientMode: 'confidential',+ clientSecret: 'sec',+ // no clientSecretExpiresAtSec → NULL (non-expiring)+ })+ await upsertDynamicClient(db, KEY, {+ serverNamespace: NS,+ serverName: 'dc-public',+ issuer: 'https://as.example.com',+ clientId: 'cid-5',+ clientMode: 'public',+ })++ const rows = await listExpiringDynamicClients(db, { withinMs: 7 * 24 * 3600 * 1000 })+ const names = rows.map(r => r.serverName).sort()+ expect(names).toEqual(['dc-expired', 'dc-expiring'])+ expect(rows.every(r => r.clientMode === 'confidential')).toBe(true)+ })+})diff --git a/control-api/test/oauth.proactiveRefreshPolicy.test.ts b/control-api/test/oauth.proactiveRefreshPolicy.test.tsnew file mode 100644index 000000000..af6a3df7a--- /dev/null+++ b/control-api/test/oauth.proactiveRefreshPolicy.test.ts@@ -0,0 +1,277 @@+/**+ * Pure decision layer for the proactive refresh + DCR lifecycle sweep+ * (mini-spec L §4/§5). Property-based (T2) plus the exact table cases (P1–P7,+ * C1–C5) and the `getAccessToken` result mapping.+ */+import { describe, expect, it } from 'vitest'+import fc from 'fast-check'+import {+ classifyDcrSecretDecision,+ classifyProactiveTokenDecision,+ resultToOutcome,+} from '../src/oauth/proactiveRefreshPolicy.js'+import type { GetAccessTokenResult } from '../src/oauth/tokenHelper.js'++const NOW = 1_700_000_000_000+const BR = 60_000 // reactive buffer+const BP = 300_000 // proactive buffer (Bp > Br)++const at = (offsetMs: number) => new Date(NOW + offsetMs)++describe('classifyProactiveTokenDecision — §4 table cases', () => {+ const eligible = { hasRefreshToken: true, supportsRefresh: true, eligibleForBackground: true }++ it('P1 NULL expiry → noop (no computable threshold)', () => {+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: null }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'null_expiry' })+ })++ it('P2 healthy (remaining > Bp) → noop', () => {+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(BP + 1) }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'healthy' })+ })++ it('P3 in-window, refreshable → refresh (inclusive upper bound Bp)', () => {+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(BP) }, NOW, BP, BR)+ ).toEqual({ kind: 'refresh' })+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(BR + 1) }, NOW, BP, BR)+ ).toEqual({ kind: 'refresh' })+ })++ it('P4 in-window, supportsRefresh=false → reconsent_imminent (fail-closed, no refresh)', () => {+ expect(+ classifyProactiveTokenDecision(+ { ...eligible, supportsRefresh: false, accessTokenExpiresAt: at(BP) },+ NOW,+ BP,+ BR+ )+ ).toEqual({ kind: 'reconsent_imminent', reason: 'no_refresh_support' })+ })++ it('P5 in-window, no refresh token → reconsent_imminent', () => {+ expect(+ classifyProactiveTokenDecision(+ { ...eligible, hasRefreshToken: false, accessTokenExpiresAt: at(BP) },+ NOW,+ BP,+ BR+ )+ ).toEqual({ kind: 'reconsent_imminent', reason: 'no_refresh_token' })+ })++ it('P6 already expired (remaining ≤ 0) → noop (not proactive work)', () => {+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(0) }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'expired' })+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(-1) }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'expired' })+ })++ it('P7 not eligible for background → noop, whatever else holds (the "cualquiera" gate)', () => {+ expect(+ classifyProactiveTokenDecision(+ { ...eligible, eligibleForBackground: false, accessTokenExpiresAt: at(BP) },+ NOW,+ BP,+ BR+ )+ ).toEqual({ kind: 'noop', reason: 'not_eligible' })+ })++ it('reactive window (0 < remaining ≤ Br) is left to the reactive path → noop', () => {+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(BR) }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'reactive_window' })+ expect(+ classifyProactiveTokenDecision({ ...eligible, accessTokenExpiresAt: at(1) }, NOW, BP, BR)+ ).toEqual({ kind: 'noop', reason: 'reactive_window' })+ })+})++describe('classifyProactiveTokenDecision — properties (T2)', () => {+ interface ArbInput {+ offsetMs: number | null+ hasRefreshToken: boolean+ supportsRefresh: boolean+ eligibleForBackground: boolean+ }+ const arbInput: fc.Arbitrary<ArbInput> = fc.record({+ offsetMs: fc.oneof(fc.constant<null>(null), fc.integer({ min: -1_000_000, max: 1_000_000 })),+ hasRefreshToken: fc.boolean(),+ supportsRefresh: fc.boolean(),+ eligibleForBackground: fc.boolean(),+ })+ // Bp > Br always (config invariant): br in range, bp = br + delta.+ const arbBuffers = fc+ .tuple(fc.integer({ min: 1, max: 500_000 }), fc.integer({ min: 1, max: 500_000 }))+ .map(([br, delta]) => ({ br, bp: br + delta }))++ const build = (i: ArbInput) => ({+ accessTokenExpiresAt: i.offsetMs === null ? null : at(i.offsetMs),+ hasRefreshToken: i.hasRefreshToken,+ supportsRefresh: i.supportsRefresh,+ eligibleForBackground: i.eligibleForBackground,+ })++ it('never refreshes outside the strict proactive window, and only when fully refreshable', () => {+ fc.assert(+ fc.property(arbInput, arbBuffers, (i, { bp, br }) => {+ const input = build(i)+ const d = classifyProactiveTokenDecision(input, NOW, bp, br)+ if (d.kind !== 'refresh') return+ expect(input.accessTokenExpiresAt).not.toBeNull()+ const remaining = (input.accessTokenExpiresAt as Date).getTime() - NOW+ expect(remaining).toBeGreaterThan(br)+ expect(remaining).toBeLessThanOrEqual(bp)+ expect(input.eligibleForBackground).toBe(true)+ expect(input.supportsRefresh).toBe(true)+ expect(input.hasRefreshToken).toBe(true)+ })+ )+ })++ it('remaining ≤ 0 is never a refresh', () => {+ fc.assert(+ fc.property(arbInput, arbBuffers, (i, { bp, br }) => {+ const input = build(i)+ if (input.accessTokenExpiresAt === null) return+ if (input.accessTokenExpiresAt.getTime() - NOW > 0) return+ expect(classifyProactiveTokenDecision(input, NOW, bp, br).kind).not.toBe('refresh')+ })+ )+ })++ it('idempotence: after a simulated full-life refresh, reclassifying is a noop', () => {+ fc.assert(+ fc.property(arbInput, arbBuffers, (i, { bp, br }) => {+ const input = build(i)+ const first = classifyProactiveTokenDecision(input, NOW, bp, br)+ if (first.kind !== 'refresh') return+ // A successful refresh pushes expiry a full life into the future (>> Bp).+ const renewed = { ...input, accessTokenExpiresAt: at(bp * 10 + 1) }+ expect(classifyProactiveTokenDecision(renewed, NOW, bp, br)).toEqual({+ kind: 'noop',+ reason: 'healthy',+ })+ })+ )+ })+})++describe('resultToOutcome — §4 result mapping', () => {+ const cases: [GetAccessTokenResult, string][] = [+ [{ kind: 'ok', accessToken: 'X' }, 'ok'],+ [{ kind: 'no_grant' }, 'no_grant'],+ [{ kind: 'refresh_failed', status: 400, detail: '' }, 'client_invalid'],+ [{ kind: 'refresh_failed', status: 401, detail: '' }, 'client_invalid'],+ [{ kind: 'refresh_failed', status: 500, detail: '' }, 'transient'],+ [{ kind: 'refresh_failed', detail: 'timeout' }, 'transient'],+ [{ kind: 'recipe_not_found' }, 'error'],+ [{ kind: 'unknown_oauth_client' }, 'error'],+ [{ kind: 'unsupported_provider', provider: 'x' }, 'error'],+ [{ kind: 'secret_missing', secret: 's' }, 'error'],+ ]+ it.each(cases)('%o → %s', (result, expected) => {+ expect(resultToOutcome(result)).toBe(expected)+ })+})++describe('classifyDcrSecretDecision — §5 table cases', () => {+ const WC = 7 * 24 * 60 * 60 * 1000++ it('C1 public → noop', () => {+ expect(+ classifyDcrSecretDecision({ clientMode: 'public', clientSecretExpiresAt: at(1000) }, NOW, WC)+ ).toEqual({ kind: 'noop', reason: 'public' })+ })++ it('C2 confidential, NULL expiry → noop', () => {+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: null },+ NOW,+ WC+ )+ ).toEqual({ kind: 'noop', reason: 'no_expiry' })+ })++ it('C3 healthy (exp > now + Wc) → noop', () => {+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(WC + 1) },+ NOW,+ WC+ )+ ).toEqual({ kind: 'noop', reason: 'healthy' })+ })++ it('C4 expiring (now < exp ≤ now + Wc) → warn', () => {+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(WC) },+ NOW,+ WC+ )+ ).toEqual({ kind: 'warn' })+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(1) },+ NOW,+ WC+ )+ ).toEqual({ kind: 'warn' })+ })++ it('C5 expired (exp ≤ now) → expired', () => {+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(0) },+ NOW,+ WC+ )+ ).toEqual({ kind: 'expired' })+ expect(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(-1) },+ NOW,+ WC+ )+ ).toEqual({ kind: 'expired' })+ })++ it('monotonicity: a later expiry is never more severe than an earlier one', () => {+ const severity = (d: ReturnType<typeof classifyDcrSecretDecision>) =>+ d.kind === 'expired' ? 2 : d.kind === 'warn' ? 1 : 0+ fc.assert(+ fc.property(+ fc.integer({ min: -1_000_000_000, max: 1_000_000_000 }),+ fc.integer({ min: -1_000_000_000, max: 1_000_000_000 }),+ fc.integer({ min: 1, max: 1_000_000_000 }),+ (o1, o2, wc) => {+ const [earlier, later] = o1 <= o2 ? [o1, o2] : [o2, o1]+ const sEarlier = severity(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(earlier) },+ NOW,+ wc+ )+ )+ const sLater = severity(+ classifyDcrSecretDecision(+ { clientMode: 'confidential', clientSecretExpiresAt: at(later) },+ NOW,+ wc+ )+ )+ expect(sLater).toBeLessThanOrEqual(sEarlier)+ }+ )+ )+ })+})diff --git a/control-api/test/services.oauthProactiveRefreshCron.test.ts b/control-api/test/services.oauthProactiveRefreshCron.test.tsnew file mode 100644index 000000000..c8d74c86e--- /dev/null+++ b/control-api/test/services.oauthProactiveRefreshCron.test.ts@@ -0,0 +1,218 @@+/**+ * Orchestration of the proactive-refresh sweep (mini-spec L §6): enumerate →+ * claim → refresh → classify, best-effort per row, plus the per-server-CR DCR+ * dedup (§5 idempotency) and the time-based DCR sweep. The DB/K8s/refresh IO is+ * injected so this asserts the OBSERVABLE sweep summary (T4), not internal calls.+ *+ * The store enumeration/claim and `getAccessToken` are mocked as control-flow+ * seams; the refresh contract itself (token-endpoint → persisted row) is proven+ * by the real-Postgres suite and the reactive `tokenHelper` tests.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import type { DbClient } from '../src/db.js'+import type { K8sGateway } from '../src/k8s.js'+import type { ExpiringDynamicClient } from '../src/oauth/dynamicClientStore.js'+import { listExpiringDynamicClients } from '../src/oauth/dynamicClientStore.js'+import type { OAuthGrantKey } from '../src/oauth/store.js'+import {+ claimRemoteGrantForRefresh,+ listRemoteGrantsInProactiveWindow,+} from '../src/oauth/store.js'+import type { GetAccessTokenResult } from '../src/oauth/tokenHelper.js'+import { getAccessToken } from '../src/oauth/tokenHelper.js'+import {+ type ProactiveRefreshSweepDeps,+ runProactiveRefreshSweep,+ startOauthProactiveRefreshCron,+ stopOauthProactiveRefreshCron,+} from '../src/services/oauthProactiveRefreshCron.js'++vi.mock('../src/oauth/store.js', async () => {+ const actual =+ await vi.importActual<typeof import('../src/oauth/store.js')>('../src/oauth/store.js')+ return {+ ...actual,+ listRemoteGrantsInProactiveWindow: vi.fn(),+ claimRemoteGrantForRefresh: vi.fn(),+ }+})+vi.mock('../src/oauth/dynamicClientStore.js', async () => {+ const actual = await vi.importActual<typeof import('../src/oauth/dynamicClientStore.js')>(+ '../src/oauth/dynamicClientStore.js'+ )+ return { ...actual, listExpiringDynamicClients: vi.fn() }+})+vi.mock('../src/oauth/tokenHelper.js', async () => {+ const actual = await vi.importActual<typeof import('../src/oauth/tokenHelper.js')>(+ '../src/oauth/tokenHelper.js'+ )+ return { ...actual, getAccessToken: vi.fn() }+})++const OPTS = { proactiveBufferMs: 300_000, reactiveBufferMs: 60_000, dcrWarnMs: 604_800_000 }+const gateway = {} as K8sGateway++const fakeTx: DbClient = { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) }+const deps: ProactiveRefreshSweepDeps = {+ db: { query: vi.fn(async () => ({ rows: [], rowCount: 0 })) },+ runInTransaction: work => work(fakeTx),+ encryptionKey: Buffer.alloc(32),+ fetchFn: vi.fn() as unknown as typeof fetch,+}++const sharedKey = (server: string, ctx: string): OAuthGrantKey => ({+ grantKind: 'shared',+ ownerKind: 'mcpserver',+ recipeNamespace: 'mcp-server',+ recipeName: server,+ contextId: ctx,+ oauthClientId: 'cid',+})++const mockedEnumerate = vi.mocked(listRemoteGrantsInProactiveWindow)+const mockedClaim = vi.mocked(claimRemoteGrantForRefresh)+const mockedGetToken = vi.mocked(getAccessToken)+const mockedDcrList = vi.mocked(listExpiringDynamicClients)++beforeEach(() => {+ vi.clearAllMocks()+ mockedEnumerate.mockResolvedValue([])+ mockedClaim.mockResolvedValue(true)+ mockedGetToken.mockResolvedValue({ kind: 'ok', accessToken: 'X' } as GetAccessTokenResult)+ mockedDcrList.mockResolvedValue([])+})++afterEach(() => {+ stopOauthProactiveRefreshCron()+})++describe('runProactiveRefreshSweep — token candidates', () => {+ it('empty candidates → all-zero summary, DCR sweep still runs', async () => {+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.candidates).toBe(0)+ expect(summary.outcomes).toEqual({+ ok: 0,+ transient: 0,+ client_invalid: 0,+ no_grant: 0,+ skipped: 0,+ error: 0,+ })+ expect(mockedDcrList).toHaveBeenCalledOnce()+ })++ it('claimed + ok → outcome ok, and the refresh runs on the transaction client', async () => {+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1')])+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.outcomes.ok).toBe(1)+ // requireBackground:true (SEC-5) and refreshBufferMs = Bp, db = the tx client.+ expect(mockedGetToken).toHaveBeenCalledWith(+ expect.objectContaining({ grantKind: 'shared', requireBackground: true }),+ expect.objectContaining({ db: fakeTx, refreshBufferMs: OPTS.proactiveBufferMs })+ )+ })++ it('claim returns false (locked / renewed out of window) → skipped, no refresh', async () => {+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1')])+ mockedClaim.mockResolvedValue(false)+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.outcomes.skipped).toBe(1)+ expect(mockedGetToken).not.toHaveBeenCalled()+ })++ it('a per-row throw is counted as error and never aborts the sweep', async () => {+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1'), sharedKey('srv-b', 'ctx-2')])+ mockedGetToken+ .mockRejectedValueOnce(new Error('boom'))+ .mockResolvedValueOnce({ kind: 'ok', accessToken: 'X' } as GetAccessTokenResult)+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.outcomes.error).toBe(1)+ expect(summary.outcomes.ok).toBe(1)+ expect(summary.candidates).toBe(2)+ })++ it('transient refresh failure → transient outcome (row left to the reactive path)', async () => {+ mockedEnumerate.mockResolvedValue([sharedKey('srv-a', 'ctx-1')])+ mockedGetToken.mockResolvedValue({+ kind: 'refresh_failed',+ status: 503,+ detail: 'upstream',+ } as GetAccessTokenResult)+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.outcomes.transient).toBe(1)+ expect(summary.outcomes.client_invalid).toBe(0)+ })++ it('§5 idempotency: N grants of one server-CR failing invalid_client apply DCR policy once', async () => {+ mockedEnumerate.mockResolvedValue([+ sharedKey('srv-a', 'ctx-1'),+ sharedKey('srv-a', 'ctx-2'),+ sharedKey('srv-a', 'ctx-3'),+ ])+ mockedGetToken.mockResolvedValue({+ kind: 'refresh_failed',+ status: 401,+ detail: 'invalid_client',+ } as GetAccessTokenResult)+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.outcomes.client_invalid).toBe(3)+ expect(summary.dcrClientInvalidServers).toEqual(['mcpserver|mcp-server|srv-a'])+ })+})++describe('runProactiveRefreshSweep — DCR expiry sweep', () => {+ it('splits expiring (warn) from expired via the pure decision', async () => {+ const now = Date.now()+ const rows: ExpiringDynamicClient[] = [+ {+ serverNamespace: 'mcp-server',+ serverName: 'srv-warn',+ clientMode: 'confidential',+ clientSecretExpiresAt: new Date(now + 1000),+ },+ {+ serverNamespace: 'mcp-server',+ serverName: 'srv-expired',+ clientMode: 'confidential',+ clientSecretExpiresAt: new Date(now - 1000),+ },+ ]+ mockedDcrList.mockResolvedValue(rows)+ const summary = await runProactiveRefreshSweep(gateway, OPTS, deps)+ expect(summary.dcrExpiring).toBe(1)+ expect(summary.dcrExpired).toBe(1)+ })++ it('a DCR read failure does not fail the sweep (best-effort)', async () => {+ mockedDcrList.mockRejectedValue(new Error('db down'))+ await expect(runProactiveRefreshSweep(gateway, OPTS, deps)).resolves.toBeDefined()+ })+})++describe('startOauthProactiveRefreshCron — loop lifecycle', () => {+ beforeEach(() => {+ vi.useFakeTimers({ shouldAdvanceTime: false })+ })+ afterEach(() => {+ stopOauthProactiveRefreshCron()+ vi.useRealTimers()+ })++ it('runs a sweep each interval and does not stack on double start', async () => {+ startOauthProactiveRefreshCron(gateway, { ...OPTS, intervalMs: 60_000 }, deps)+ startOauthProactiveRefreshCron(gateway, { ...OPTS, intervalMs: 60_000 }, deps)+ await vi.advanceTimersByTimeAsync(60_000)+ expect(mockedEnumerate).toHaveBeenCalledTimes(1)+ await vi.advanceTimersByTimeAsync(60_000)+ expect(mockedEnumerate).toHaveBeenCalledTimes(2)+ })++ it('stops cleanly', async () => {+ startOauthProactiveRefreshCron(gateway, { ...OPTS, intervalMs: 60_000 }, deps)+ await vi.advanceTimersByTimeAsync(60_000)+ const before = mockedEnumerate.mock.calls.length+ stopOauthProactiveRefreshCron()+ await vi.advanceTimersByTimeAsync(180_000)+ expect(mockedEnumerate.mock.calls.length).toBe(before)+ })+})