Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): let rateLimitMiddleware fail closed for the external GFS grants and shares buckets
Alfredo Lopez11b9334d4e65
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex beaecc0fd..8fa990243 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.608",+ "version": "0.1.609","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.608",+ "version": "0.1.609","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 9919d7f2e..6b17fc92c 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.608",+ "version": "0.1.609","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.tsindex f679296f6..58f9d9ebd 100644--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts@@ -32,6 +32,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {// ENV-tunable platform limit (issue #348): CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN.maxPerMinute: config.pluginSdkRequestBucketRlPerMin,getBucketKey: pluginWorkloadSdkRequestBucketKey,+ onBackendUnavailable: 'open',})}@@ -45,6 +46,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {if (!claims) return 'plugin_workload_sdk_internal:unauthenticated'return `plugin_workload_sdk_internal:${claims.iss}:${claims.sub}`},+ onBackendUnavailable: 'open',})}@@ -62,5 +64,6 @@ export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {const sub = (req as UiAuthedRequest).adminAuth?.subreturn `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`},+ onBackendUnavailable: 'open',})}diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.tsindex 6c3e62a09..47d2cbbcb 100644--- a/control-api/src/middleware/rateLimitMiddleware.ts+++ b/control-api/src/middleware/rateLimitMiddleware.ts@@ -1,6 +1,9 @@import { NextFunction, Request, Response } from 'express'import { rateLimitHitsTotal } from '../observability/metrics.js'-import { checkAndIncrement } from '../services/rateLimiterService.js'+import {+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ checkAndIncrement,+} from '../services/rateLimiterService.js'/*** Factory for a per-request rate limit middleware backed by a PG token bucket.@@ -22,11 +25,16 @@ import { checkAndIncrement } from '../services/rateLimiterService.js'* - If `getBucketKey` returns `null` (e.g. unable to derive key), the* request is *allowed* but not counted — fail-open. Callers that need* strict enforcement should return a sentinel key instead.+ * - When the limiter backend cannot count the request+ * (`backendAvailable: false`), `onBackendUnavailable` decides: `'open'`+ * lets it through uncounted, `'closed'` answers 503 with `Retry-After`.+ * The option is required so every caller states that choice.*/export function rateLimitMiddleware(opts: {bucketType: stringmaxPerMinute: numbergetBucketKey: (req: Request) => string | null+ onBackendUnavailable: 'open' | 'closed'}) {return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {void (async () => {@@ -40,6 +48,22 @@ export function rateLimitMiddleware(opts: {}const result = await checkAndIncrement(key, opts.maxPerMinute)+ if (!result.backendAvailable && opts.onBackendUnavailable === 'closed') {+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)+ if (req.log) {+ req.log.warn(+ { event: 'rate_limit_unavailable', bucketType: opts.bucketType },+ 'rate limit backend unavailable'+ )+ }+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))+ res.setHeader('Cache-Control', 'no-store')+ res.status(503).json({+ error: 'rate_limit_unavailable',+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ })+ return+ }if (!result.allowed) {rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 44b326f20..1d2737f93 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -250,7 +250,7 @@ export const approvalsArchiveDurationSeconds = getOrCreateHistogram({// ─── Rate limiter metrics ─────────────────────────────────────────────────export const rateLimitHitsTotal = getOrCreateCounter({name: 'rate_limit_hits_total',- help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied).',+ help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable).',labelNames: ['bucket_type', 'result'] as const as Array<'bucket_type' | 'result'>,})diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.tsindex e99b22587..e8b976352 100644--- a/control-api/src/routes/admin/auth.ts+++ b/control-api/src/routes/admin/auth.ts@@ -63,6 +63,7 @@ function publicAdminTokenRateLimit(routeName: string) {const remote = req.ip || req.socket.remoteAddress || 'unknown'return `control-admin-public:${routeName}:${remote}:${email || login || token.slice(0, 48) || 'missing'}`},+ onBackendUnavailable: 'open',})}diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex a477fd59c..eee584e8b 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -3166,6 +3166,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orgkeys:${sub}` : null},+ onBackendUnavailable: 'open',})// Guards ⓪①② — returns null after sending an error response.@@ -3287,6 +3288,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orggrants:${sub}` : null},+ onBackendUnavailable: 'open',})// Resolve the caller's own org for a self-service op, or send the terminal errordiff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.tsindex f3b82db36..25357ad1f 100644--- a/control-api/src/routes/admin/registryConnect.ts+++ b/control-api/src/routes/admin/registryConnect.ts@@ -223,6 +223,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_status:${sub}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {@@ -333,6 +334,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_request:${sub}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {@@ -591,6 +593,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_recover:${sub}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.tsindex eb633acf5..e50c500ad 100644--- a/control-api/src/routes/external/auth.ts+++ b/control-api/src/routes/external/auth.ts@@ -129,6 +129,7 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {.toLowerCase()return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/decisions.routes.ts b/control-api/src/routes/external/decisions.routes.tsindex 4b51b539b..4967cd6c8 100644--- a/control-api/src/routes/external/decisions.routes.ts+++ b/control-api/src/routes/external/decisions.routes.ts@@ -63,6 +63,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}` : null},+ onBackendUnavailable: 'open',}),(req, res, next) => {void (async () => {@@ -102,6 +103,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}` : null},+ onBackendUnavailable: 'open',}),requireApprovalDecisionAccess(),(req, res, next) => {diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.tsindex d6befdfb5..4deabc3e5 100644--- a/control-api/src/routes/external/gfs.ts+++ b/control-api/src/routes/external/gfs.ts@@ -503,6 +503,9 @@ export function createExternalGfsRouter(): Router {}return 'gfsgrants-ext-read:__no_authority__'},+ // Like the Postgres buckets above, an uncountable request is refused+ // rather than admitted unmetered (#764).+ onBackendUnavailable: 'closed',})// Keep the existing mutation bucket identity and 30/min budget for@@ -521,6 +524,7 @@ export function createExternalGfsRouter(): Router {}return 'gfsgrants-ext:__no_authority__'},+ onBackendUnavailable: 'closed',})// ── token mint (mirror /external/rpc/token, but a gfs token sub=users.id) ──diff --git a/control-api/src/routes/external/invitations.ts b/control-api/src/routes/external/invitations.tsindex c6ad106a5..a7e8c0c75 100644--- a/control-api/src/routes/external/invitations.ts+++ b/control-api/src/routes/external/invitations.ts@@ -46,6 +46,7 @@ export function createExternalInvitationsRouter(): Router {bucketType: 'external_invitation_lookup',maxPerMinute: 30,getBucketKey: req => invitationLookupIpKey(req),+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {@@ -79,6 +80,7 @@ export function createExternalInvitationsRouter(): Router {bucketType: 'external_invitation_password_token',maxPerMinute: 10,getBucketKey: req => invitationLookupIpKey(req),+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/notifications.routes.ts b/control-api/src/routes/external/notifications.routes.tsindex ae30dc9bb..23f25adfe 100644--- a/control-api/src/routes/external/notifications.routes.ts+++ b/control-api/src/routes/external/notifications.routes.ts@@ -111,6 +111,7 @@ export function createExternalNotificationsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}:notifications` : null},+ onBackendUnavailable: 'open',}),(req: Request, res: Response) => {void (async () => {@@ -344,6 +345,7 @@ export function createExternalNotificationsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}:notification-ack` : null},+ onBackendUnavailable: 'open',}),asyncHandler(async (req: Request, res: Response) => {const extReq = req as ExternalAuthedRequestdiff --git a/control-api/src/routes/external/users.ts b/control-api/src/routes/external/users.tsindex 8be8d9540..5378b7ca8 100644--- a/control-api/src/routes/external/users.ts+++ b/control-api/src/routes/external/users.ts@@ -107,6 +107,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {const userId = (req as ExternalAuthedRequest).externalAuth?.userIdreturn userId ? `team_directory:${userId}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {@@ -284,6 +285,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {const userId = (req as ExternalAuthedRequest).externalAuth?.userIdreturn userId ? `profile_password_update:${userId}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.tsindex a745cc9ab..007aa15ef 100644--- a/control-api/src/routes/gfs/grants.ts+++ b/control-api/src/routes/gfs/grants.ts@@ -896,6 +896,7 @@ export function registerGfsGrantRoutes(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants:${sub}` : null},+ onBackendUnavailable: 'open',})router.get('/gfs/grants', requireAuthForControlUI, grantsRateLimit, asyncHandler(handleGrantRead))router.put(diff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.tsindex 9abe85b03..4586ea55f 100644--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts@@ -69,6 +69,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants-legacy:${sub}` : null},+ onBackendUnavailable: 'open',})router.get('/gfs/grants/legacy-standalone',diff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.tsindex 5036e3ed7..e16c98697 100644--- a/control-api/src/routes/gfs/shares.ts+++ b/control-api/src/routes/gfs/shares.ts@@ -45,6 +45,7 @@ export function registerGfsShareRoutes(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsshares:${sub}` : null},+ onBackendUnavailable: 'open',})router.get('/gfs/shares', requireAuthForControlUI, sharesRateLimit, asyncHandler(handleShareRead))router.post(diff --git a/control-api/src/routes/internal/oauth.ts b/control-api/src/routes/internal/oauth.tsindex 4b76c8013..f6b247c96 100644--- a/control-api/src/routes/internal/oauth.ts+++ b/control-api/src/routes/internal/oauth.ts@@ -77,6 +77,7 @@ export function createInternalOAuthRouter(gateway: K8sGateway): Router {: (req.internalService?.name ?? 'unknown')return `mcp-oauth-internal:${uid}`},+ onBackendUnavailable: 'open',})const recipeReader: RecipeReader = {diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.tsindex f9f877f3f..21c56d2f6 100644--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts@@ -82,6 +82,7 @@ const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({bucketType: 'plugin_workload_sdk_credential',maxPerMinute: 120,getBucketKey: pluginWorkloadSdkCredentialBucketKey,+ onBackendUnavailable: 'open',})function requirePluginWorkloadSdkScope(req: Request, res: Response, next: NextFunction): void {diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.tsindex 0a710a645..80d3f72f5 100644--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts@@ -31,6 +31,7 @@ export function createMcpHostRefreshRoutes(): Router {const key = getMcpHostRefreshRateLimitKey(token)return key ? `refresh:${key}` : null},+ onBackendUnavailable: 'open',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts b/control-api/src/routes/mcp-host/renewal/reissue.routes.tsindex afb7bdbac..c84614eaf 100644--- a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts+++ b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts@@ -37,6 +37,7 @@ export function createMcpHostReissueRoutes(): Router {const key = getMcpHostExpiredRefreshRateLimitKey(token)return key ? `reissue:${key}` : null},+ onBackendUnavailable: 'open',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.tsindex 36a8f11ee..763dc71e2 100644--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts@@ -491,6 +491,7 @@ export function createUserApprovalRequestsRoutes(if (!auth) return nullreturn mcpHostRateLimitBucketKey('recipe', auth)},+ onBackendUnavailable: 'open',}),(req, res, next) => {void (async () => {@@ -1388,6 +1389,7 @@ export function createUserApprovalRequestsRoutes(const approvalId = String(req.params?.id || '').trim()return approvalId ? `provider-decision:${approvalId}` : null},+ onBackendUnavailable: 'open',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcpOauth.ts b/control-api/src/routes/mcpOauth.tsindex c5ee97884..909d72b13 100644--- a/control-api/src/routes/mcpOauth.ts+++ b/control-api/src/routes/mcpOauth.ts@@ -378,6 +378,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefinedreturn claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'},+ onBackendUnavailable: 'open',}),async (req, res, next) => {const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined@@ -542,6 +543,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefinedreturn claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'},+ onBackendUnavailable: 'open',}),async (req, res, next) => {const claims = res.locals.mcpHostControl as McpHostControlClaims | undefineddiff --git a/control-api/src/routes/recipeOauth.ts b/control-api/src/routes/recipeOauth.tsindex c933c4343..60fb8877d 100644--- a/control-api/src/routes/recipeOauth.ts+++ b/control-api/src/routes/recipeOauth.ts@@ -81,6 +81,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth:${req.recipe.namespace}/${req.recipe.name}` : null,+ onBackendUnavailable: 'open',}),async (req, res, next) => {const { namespace: recipeNamespace, name: recipeName } = req.recipe!@@ -189,6 +190,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth-user:${req.recipe.namespace}/${req.recipe.name}` : null,+ onBackendUnavailable: 'open',}),async (req, res, next) => {// [SEC-5] recipe identity ONLY from the broker token sub; userId is a@@ -198,8 +200,10 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {const oauthClientId = (req.body ?? {}).oauthClientIdconst userId = (req.body ?? {}).userIdif (- typeof oauthClientId !== 'string' || oauthClientId.length === 0 ||- typeof userId !== 'string' || userId.length === 0+ typeof oauthClientId !== 'string' ||+ oauthClientId.length === 0 ||+ typeof userId !== 'string' ||+ userId.length === 0) {return res.status(400).json({ error: 'invalid_request' })}@@ -220,7 +224,14 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {}const result = await getAccessToken(- { grantKind: 'user', recipeNamespace, recipeName, userId, oauthClientId, requireBackground: true },+ {+ grantKind: 'user',+ recipeNamespace,+ recipeName,+ userId,+ oauthClientId,+ requireBackground: true,+ },{db: { query: (text, values) => pool.query(text, values) },recipeReader,@@ -232,7 +243,14 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {const audit = (outcome: string) =>req.log?.info(- { event: 'recipe_oauth_user_token_issued', recipeNamespace, recipeName, userId, oauthClientId, outcome },+ {+ event: 'recipe_oauth_user_token_issued',+ recipeNamespace,+ recipeName,+ userId,+ oauthClientId,+ outcome,+ },'recipe oauth user token issued')@@ -254,13 +272,17 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {return res.status(400).json({ error: 'unknown_oauth_client' })case 'unsupported_provider':audit('unsupported_provider')- return res.status(400).json({ error: 'unsupported_provider', provider: result.provider })+ return res+ .status(400)+ .json({ error: 'unsupported_provider', provider: result.provider })case 'secret_missing':audit('integration_not_configured')return res.status(503).json(integrationNotConfigured(oauthClientId, result.secret))case 'refresh_failed':audit('refresh_failed')- return res.status(502).json({ error: 'refresh_failed', status: result.status, detail: result.detail })+ return res+ .status(502)+ .json({ error: 'refresh_failed', status: result.status, detail: result.detail })}} catch (err) {next(err)@@ -283,6 +305,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth-users:${req.recipe.namespace}/${req.recipe.name}` : null,+ onBackendUnavailable: 'open',}),async (req, res, next) => {// [SEC-6] recipe identity from the broker token sub; returns only THIS@@ -314,7 +337,13 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {)req.log?.info(- { event: 'recipe_oauth_users_listed', recipeNamespace, recipeName, oauthClientId, count: users.length },+ {+ event: 'recipe_oauth_users_listed',+ recipeNamespace,+ recipeName,+ oauthClientId,+ count: users.length,+ },'recipe oauth users listed')return res.status(200).json({ users })diff --git a/control-api/src/routes/registry.ts b/control-api/src/routes/registry.tsindex b6b682bad..a698a5dd1 100644--- a/control-api/src/routes/registry.ts+++ b/control-api/src/routes/registry.ts@@ -32,6 +32,7 @@ export function createRegistryRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `voucher:${sub}` : null},+ onBackendUnavailable: 'open',}),async (req: UiAuthedRequest, res, next) => {try {diff --git a/control-api/src/routes/rpc-access/hosts.ts b/control-api/src/routes/rpc-access/hosts.tsindex 96d3c59fd..92b2b04be 100644--- a/control-api/src/routes/rpc-access/hosts.ts+++ b/control-api/src/routes/rpc-access/hosts.ts@@ -61,6 +61,7 @@ export function createRpcAccessHostsRouter(gateway: K8sGateway): Router {const hostRef = String(req.params?.hostRef || '').trim()return hostRef ? `host-wake:${hostRef}` : null},+ onBackendUnavailable: 'open',}),async (req, res, next) => {const hostRef = String(req.params.hostRef || '').trim()diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.tsindex 18e5a2553..42a04e36f 100644--- a/control-api/src/routes/workflows/shared/rateLimit.ts+++ b/control-api/src/routes/workflows/shared/rateLimit.ts@@ -152,6 +152,7 @@ function adminCodexReadRateLimit() {bucketType: 'admin_codex_read',maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),+ onBackendUnavailable: 'open',})}@@ -160,6 +161,7 @@ function adminCodexWriteRateLimit() {bucketType: 'admin_codex_write',maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),+ onBackendUnavailable: 'open',})}@@ -194,6 +196,7 @@ export function codexOAuthCallbackRateLimits() {bucketType: 'codex_oauth_callback',maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,getBucketKey: codexOAuthCallbackBucketKey,+ onBackendUnavailable: 'open',}),] as const}@@ -231,6 +234,7 @@ export function llmProviderAttemptAuthorizeRateLimits() {bucketType: 'llm_provider_attempt_authorize',maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,getBucketKey: mcpHostAttemptRateLimitKey,+ onBackendUnavailable: 'open',}),] as const}@@ -251,6 +255,7 @@ export function adminWorkflowTriggerRateLimit() {bucketType: 'workflow_trigger',maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,getBucketKey: adminWorkflowTriggerRateLimitKey,+ onBackendUnavailable: 'open',})}@@ -268,6 +273,7 @@ function workflowTriggerRateLimitFor(getCredential: (req: Request) => string | nif (!credential) return nullreturn hashedWorkflowTriggerBucket(credential)},+ onBackendUnavailable: 'open',})}@@ -326,6 +332,7 @@ function workflowAdminReadRateLimit() {bucketType: 'workflow_admin_read',maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),+ onBackendUnavailable: 'open',})}@@ -334,6 +341,7 @@ function adminOutputsReadRateLimit() {bucketType: 'admin_outputs_read',maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),+ onBackendUnavailable: 'open',})}@@ -342,6 +350,7 @@ export function workflowGrantReadRateLimit() {bucketType: 'workflow_grants_read',maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),+ onBackendUnavailable: 'open',})}@@ -350,6 +359,7 @@ export function workflowGrantWriteRateLimit() {bucketType: 'workflow_grants_write',maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),+ onBackendUnavailable: 'open',})}diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.tsnew file mode 100644index 000000000..e55b1424c--- /dev/null+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts@@ -0,0 +1,118 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import type { RateLimitCheck } from '../src/services/rateLimiterService.js'++const checkAndIncrement = vi.hoisted(() => vi.fn())+const hits = vi.hoisted(() => ({ inc: vi.fn() }))++vi.mock('../src/services/rateLimiterService.js', () => ({+ checkAndIncrement,+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,+}))+vi.mock('../src/observability/metrics.js', () => ({ rateLimitHitsTotal: hits }))++const { rateLimitMiddleware } = await import('../src/middleware/rateLimitMiddleware.js')++const LIMIT = 5+const resetMs = Date.now() + 30_000++function available(count: number): RateLimitCheck {+ return {+ allowed: count <= LIMIT,+ remaining: Math.max(0, LIMIT - count),+ resetMs,+ windowStartMs: resetMs - 60_000,+ count,+ backendAvailable: true,+ }+}++/** What checkAndIncrement returns when it could not count the request. */+function unavailable(): RateLimitCheck {+ return {+ allowed: true,+ remaining: LIMIT,+ resetMs,+ windowStartMs: resetMs - 60_000,+ count: 0,+ backendAvailable: false,+ }+}++function appWith(onBackendUnavailable: 'open' | 'closed') {+ const handler = vi.fn((_req: express.Request, res: express.Response) => {+ res.status(204).end()+ })+ const app = express()+ app.get(+ '/limited',+ rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'unit:key',+ onBackendUnavailable,+ }),+ handler+ )+ return { app, handler }+}++describe('rateLimitMiddleware backend-unavailable policy', () => {+ beforeEach(() => {+ checkAndIncrement.mockReset()+ hits.inc.mockReset()+ })++ it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {+ checkAndIncrement.mockResolvedValueOnce(unavailable())+ const { app, handler } = appWith('closed')++ const response = await request(app).get('/limited')++ expect(response.status).toBe(503)+ expect(response.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })+ expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['cache-control']).toBe('no-store')+ // Liveness witness: the limiter was consulted with the caller's key.+ expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)+ expect(hits.inc).toHaveBeenCalledExactlyOnceWith(+ { bucket_type: 'unit_bucket', result: 'unavailable' },+ 1+ )+ expect(handler).not.toHaveBeenCalled()+ })++ it("'open' lets the uncounted request through, as before", async () => {+ checkAndIncrement.mockResolvedValueOnce(unavailable())+ const { app, handler } = appWith('open')++ const response = await request(app).get('/limited')++ expect(response.status).toBe(204)+ expect(handler).toHaveBeenCalledTimes(1)+ expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)+ expect(hits.inc).toHaveBeenCalledExactlyOnceWith(+ { bucket_type: 'unit_bucket', result: 'allowed' },+ 1+ )+ })++ it("'closed' still admits a counted request and still answers 429 over the limit", async () => {+ checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))+ const { app, handler } = appWith('closed')++ const admitted = await request(app).get('/limited')+ const denied = await request(app).get('/limited')++ expect(admitted.status).toBe(204)+ expect(admitted.headers['x-ratelimit-remaining']).toBe('4')+ expect(denied.status).toBe(429)+ expect(denied.body).toMatchObject({ error: 'Too Many Requests' })+ expect(handler).toHaveBeenCalledTimes(1)+ expect(hits.inc.mock.calls).toEqual([+ [{ bucket_type: 'unit_bucket', result: 'allowed' }, 1],+ [{ bucket_type: 'unit_bucket', result: 'denied' }, 1],+ ])+ })+})diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.tsindex 82e18e0b1..f501ae2ca 100644--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts@@ -65,7 +65,6 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =let mod: {createApp: (gateway: unknown) => import('express').Expressconfig: typeof import('../src/config.js').config- rootLogger: typeof import('../src/observability/logger.js').rootLoggerrequestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotalMockGateway: typeof import('./mockGateway.js').MockGateway}@@ -94,13 +93,11 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =const appMod = await import('../src/app.js')const configMod = await import('../src/config.js')- const loggerMod = await import('../src/observability/logger.js')const metricsMod = await import('../src/observability/metrics.js')const { MockGateway } = await import('./mockGateway.js')mod = {createApp: appMod.createApp as never,config: configMod.config,- rootLogger: loggerMod.rootLogger,requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,MockGateway,}@@ -229,4 +226,72 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =},T3_TIMEOUT_MS)++ // Holding the pool (T3) cannot reach the grants rateLimitMiddleware: the+ // pre-resolution Postgres bucket fails closed first. A trigger instead+ // fails every limiter upsert whose key is not an external GFS Postgres+ // bucket, so those keep counting and the grants bucket gets a real query+ // error (plan addendum A2).+ async function injectLimiterFault(): Promise<void> {+ await corePool.query(`+ CREATE FUNCTION t3b_limiter_fault() RETURNS trigger LANGUAGE plpgsql AS $$+ BEGIN+ IF NEW.bucket_key NOT LIKE 'gfs-ext:%' THEN+ RAISE EXCEPTION 't3b injected limiter fault';+ END IF;+ RETURN NEW;+ END+ $$`)+ await corePool.query(`+ CREATE TRIGGER t3b_limiter_fault BEFORE INSERT ON rate_limit_buckets+ FOR EACH ROW EXECUTE FUNCTION t3b_limiter_fault()`)+ }++ async function removeLimiterFault(): Promise<void> {+ await corePool.query('DROP TRIGGER IF EXISTS t3b_limiter_fault ON rate_limit_buckets')+ await corePool.query('DROP FUNCTION IF EXISTS t3b_limiter_fault()')+ }++ it('T3b: the external grants limiter fails closed on a backend error while an open-policy route still passes', async () => {+ const app = mod.createApp(new mod.MockGateway())+ const userId = await seedUser()+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ const external = (req: request.Test) =>+ req+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', JSON.stringify({ userId }))+ .set('x-forwarded-for', '203.0.113.7')+ await corePool.query('DELETE FROM rate_limit_buckets')++ let grants: request.Response+ let ack: request.Response+ await injectLimiterFault()+ try {+ grants = await external(+ request(app).get(`/api/v1/external/gfs/grants?drive=main&resourceId=${randomUUID()}`)+ )+ ack = await external(request(app).post(`/api/v1/external/notifications/${randomUUID()}/ack`))+ } finally {+ await removeLimiterFault()+ }++ // The 503 comes from rateLimitMiddleware, not from the external GFS+ // Postgres buckets (whose body is gfs_rate_limit_unavailable).+ expect(grants.status).toBe(503)+ expect(grants.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })+ expect(grants.headers['retry-after']).toBe('2')+ expect(grants.headers['cache-control']).toBe('no-store')+ // Witness for the open policy under the same fault: the ack route's+ // limiter could not count either, and its handler still answered.+ expect(ack.status).toBe(404)+ expect(ack.body).toEqual({ error: 'notification_not_found' })++ // Witness that the fault is scoped as intended: the external GFS Postgres+ // buckets counted, and nothing else reached the ledger.+ const keys = await ledgerKeys()+ expect(keys.length).toBeGreaterThan(0)+ expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)+ }, 30_000)})