Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): let rateLimitMiddleware fail closed for the external GFS grants and shares buckets

Alfredo Lopez11b9334d4e65
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index beaecc0fd..8fa990243 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.608",
+ "version": "0.1.609",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.608",
+ "version": "0.1.609",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 9919d7f2e..6b17fc92c 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.608",
+ "version": "0.1.609",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
index f679296f6..58f9d9ebd 100644
--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
@@ -32,6 +32,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {
// ENV-tunable platform limit (issue #348): CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN.
maxPerMinute: config.pluginSdkRequestBucketRlPerMin,
getBucketKey: pluginWorkloadSdkRequestBucketKey,
+ onBackendUnavailable: 'open',
})
}
@@ -45,6 +46,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {
if (!claims) return 'plugin_workload_sdk_internal:unauthenticated'
return `plugin_workload_sdk_internal:${claims.iss}:${claims.sub}`
},
+ onBackendUnavailable: 'open',
})
}
@@ -62,5 +64,6 @@ export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`
},
+ onBackendUnavailable: 'open',
})
}
diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.ts
index 6c3e62a09..47d2cbbcb 100644
--- a/control-api/src/middleware/rateLimitMiddleware.ts
+++ b/control-api/src/middleware/rateLimitMiddleware.ts
@@ -1,6 +1,9 @@
import { NextFunction, Request, Response } from 'express'
import { rateLimitHitsTotal } from '../observability/metrics.js'
-import { checkAndIncrement } from '../services/rateLimiterService.js'
+import {
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ checkAndIncrement,
+} from '../services/rateLimiterService.js'
/**
* Factory for a per-request rate limit middleware backed by a PG token bucket.
@@ -22,11 +25,16 @@ import { checkAndIncrement } from '../services/rateLimiterService.js'
* - If `getBucketKey` returns `null` (e.g. unable to derive key), the
* request is *allowed* but not counted — fail-open. Callers that need
* strict enforcement should return a sentinel key instead.
+ * - When the limiter backend cannot count the request
+ * (`backendAvailable: false`), `onBackendUnavailable` decides: `'open'`
+ * lets it through uncounted, `'closed'` answers 503 with `Retry-After`.
+ * The option is required so every caller states that choice.
*/
export function rateLimitMiddleware(opts: {
bucketType: string
maxPerMinute: number
getBucketKey: (req: Request) => string | null
+ onBackendUnavailable: 'open' | 'closed'
}) {
return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {
void (async () => {
@@ -40,6 +48,22 @@ export function rateLimitMiddleware(opts: {
}
const result = await checkAndIncrement(key, opts.maxPerMinute)
+ if (!result.backendAvailable && opts.onBackendUnavailable === 'closed') {
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)
+ if (req.log) {
+ req.log.warn(
+ { event: 'rate_limit_unavailable', bucketType: opts.bucketType },
+ 'rate limit backend unavailable'
+ )
+ }
+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
+ res.setHeader('Cache-Control', 'no-store')
+ res.status(503).json({
+ error: 'rate_limit_unavailable',
+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ })
+ return
+ }
if (!result.allowed) {
rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)
const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 44b326f20..1d2737f93 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -250,7 +250,7 @@ export const approvalsArchiveDurationSeconds = getOrCreateHistogram({
// ─── Rate limiter metrics ─────────────────────────────────────────────────
export const rateLimitHitsTotal = getOrCreateCounter({
name: 'rate_limit_hits_total',
- help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied).',
+ help: 'Rate-limiter hits, labelled by bucket_type and result (allowed|denied|unavailable).',
labelNames: ['bucket_type', 'result'] as const as Array<'bucket_type' | 'result'>,
})
diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.ts
index e99b22587..e8b976352 100644
--- a/control-api/src/routes/admin/auth.ts
+++ b/control-api/src/routes/admin/auth.ts
@@ -63,6 +63,7 @@ function publicAdminTokenRateLimit(routeName: string) {
const remote = req.ip || req.socket.remoteAddress || 'unknown'
return `control-admin-public:${routeName}:${remote}:${email || login || token.slice(0, 48) || 'missing'}`
},
+ onBackendUnavailable: 'open',
})
}
diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.ts
index a477fd59c..eee584e8b 100644
--- a/control-api/src/routes/admin/registry.ts
+++ b/control-api/src/routes/admin/registry.ts
@@ -3166,6 +3166,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orgkeys:${sub}` : null
},
+ onBackendUnavailable: 'open',
})
// Guards ⓪①② — returns null after sending an error response.
@@ -3287,6 +3288,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orggrants:${sub}` : null
},
+ onBackendUnavailable: 'open',
})
// Resolve the caller's own org for a self-service op, or send the terminal error
diff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.ts
index f3b82db36..25357ad1f 100644
--- a/control-api/src/routes/admin/registryConnect.ts
+++ b/control-api/src/routes/admin/registryConnect.ts
@@ -223,6 +223,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_status:${sub}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
@@ -333,6 +334,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_request:${sub}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
@@ -591,6 +593,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_recover:${sub}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.ts
index eb633acf5..e50c500ad 100644
--- a/control-api/src/routes/external/auth.ts
+++ b/control-api/src/routes/external/auth.ts
@@ -129,6 +129,7 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {
.toLowerCase()
return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/decisions.routes.ts b/control-api/src/routes/external/decisions.routes.ts
index 4b51b539b..4967cd6c8 100644
--- a/control-api/src/routes/external/decisions.routes.ts
+++ b/control-api/src/routes/external/decisions.routes.ts
@@ -63,6 +63,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}` : null
},
+ onBackendUnavailable: 'open',
}),
(req, res, next) => {
void (async () => {
@@ -102,6 +103,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}` : null
},
+ onBackendUnavailable: 'open',
}),
requireApprovalDecisionAccess(),
(req, res, next) => {
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.ts
index d6befdfb5..4deabc3e5 100644
--- a/control-api/src/routes/external/gfs.ts
+++ b/control-api/src/routes/external/gfs.ts
@@ -503,6 +503,9 @@ export function createExternalGfsRouter(): Router {
}
return 'gfsgrants-ext-read:__no_authority__'
},
+ // Like the Postgres buckets above, an uncountable request is refused
+ // rather than admitted unmetered (#764).
+ onBackendUnavailable: 'closed',
})
// Keep the existing mutation bucket identity and 30/min budget for
@@ -521,6 +524,7 @@ export function createExternalGfsRouter(): Router {
}
return 'gfsgrants-ext:__no_authority__'
},
+ onBackendUnavailable: 'closed',
})
// ── token mint (mirror /external/rpc/token, but a gfs token sub=users.id) ──
diff --git a/control-api/src/routes/external/invitations.ts b/control-api/src/routes/external/invitations.ts
index c6ad106a5..a7e8c0c75 100644
--- a/control-api/src/routes/external/invitations.ts
+++ b/control-api/src/routes/external/invitations.ts
@@ -46,6 +46,7 @@ export function createExternalInvitationsRouter(): Router {
bucketType: 'external_invitation_lookup',
maxPerMinute: 30,
getBucketKey: req => invitationLookupIpKey(req),
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
@@ -79,6 +80,7 @@ export function createExternalInvitationsRouter(): Router {
bucketType: 'external_invitation_password_token',
maxPerMinute: 10,
getBucketKey: req => invitationLookupIpKey(req),
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/notifications.routes.ts b/control-api/src/routes/external/notifications.routes.ts
index ae30dc9bb..23f25adfe 100644
--- a/control-api/src/routes/external/notifications.routes.ts
+++ b/control-api/src/routes/external/notifications.routes.ts
@@ -111,6 +111,7 @@ export function createExternalNotificationsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}:notifications` : null
},
+ onBackendUnavailable: 'open',
}),
(req: Request, res: Response) => {
void (async () => {
@@ -344,6 +345,7 @@ export function createExternalNotificationsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}:notification-ack` : null
},
+ onBackendUnavailable: 'open',
}),
asyncHandler(async (req: Request, res: Response) => {
const extReq = req as ExternalAuthedRequest
diff --git a/control-api/src/routes/external/users.ts b/control-api/src/routes/external/users.ts
index 8be8d9540..5378b7ca8 100644
--- a/control-api/src/routes/external/users.ts
+++ b/control-api/src/routes/external/users.ts
@@ -107,6 +107,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {
const userId = (req as ExternalAuthedRequest).externalAuth?.userId
return userId ? `team_directory:${userId}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
@@ -284,6 +285,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {
const userId = (req as ExternalAuthedRequest).externalAuth?.userId
return userId ? `profile_password_update:${userId}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.ts
index a745cc9ab..007aa15ef 100644
--- a/control-api/src/routes/gfs/grants.ts
+++ b/control-api/src/routes/gfs/grants.ts
@@ -896,6 +896,7 @@ export function registerGfsGrantRoutes(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants:${sub}` : null
},
+ onBackendUnavailable: 'open',
})
router.get('/gfs/grants', requireAuthForControlUI, grantsRateLimit, asyncHandler(handleGrantRead))
router.put(
diff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
index 9abe85b03..4586ea55f 100644
--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts
+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
@@ -69,6 +69,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants-legacy:${sub}` : null
},
+ onBackendUnavailable: 'open',
})
router.get(
'/gfs/grants/legacy-standalone',
diff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.ts
index 5036e3ed7..e16c98697 100644
--- a/control-api/src/routes/gfs/shares.ts
+++ b/control-api/src/routes/gfs/shares.ts
@@ -45,6 +45,7 @@ export function registerGfsShareRoutes(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsshares:${sub}` : null
},
+ onBackendUnavailable: 'open',
})
router.get('/gfs/shares', requireAuthForControlUI, sharesRateLimit, asyncHandler(handleShareRead))
router.post(
diff --git a/control-api/src/routes/internal/oauth.ts b/control-api/src/routes/internal/oauth.ts
index 4b76c8013..f6b247c96 100644
--- a/control-api/src/routes/internal/oauth.ts
+++ b/control-api/src/routes/internal/oauth.ts
@@ -77,6 +77,7 @@ export function createInternalOAuthRouter(gateway: K8sGateway): Router {
: (req.internalService?.name ?? 'unknown')
return `mcp-oauth-internal:${uid}`
},
+ onBackendUnavailable: 'open',
})
const recipeReader: RecipeReader = {
diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
index f9f877f3f..21c56d2f6 100644
--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
@@ -82,6 +82,7 @@ const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({
bucketType: 'plugin_workload_sdk_credential',
maxPerMinute: 120,
getBucketKey: pluginWorkloadSdkCredentialBucketKey,
+ onBackendUnavailable: 'open',
})
function requirePluginWorkloadSdkScope(req: Request, res: Response, next: NextFunction): void {
diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
index 0a710a645..80d3f72f5 100644
--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
@@ -31,6 +31,7 @@ export function createMcpHostRefreshRoutes(): Router {
const key = getMcpHostRefreshRateLimitKey(token)
return key ? `refresh:${key}` : null
},
+ onBackendUnavailable: 'open',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
index afb7bdbac..c84614eaf 100644
--- a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
+++ b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
@@ -37,6 +37,7 @@ export function createMcpHostReissueRoutes(): Router {
const key = getMcpHostExpiredRefreshRateLimitKey(token)
return key ? `reissue:${key}` : null
},
+ onBackendUnavailable: 'open',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
index 36a8f11ee..763dc71e2 100644
--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
@@ -491,6 +491,7 @@ export function createUserApprovalRequestsRoutes(
if (!auth) return null
return mcpHostRateLimitBucketKey('recipe', auth)
},
+ onBackendUnavailable: 'open',
}),
(req, res, next) => {
void (async () => {
@@ -1388,6 +1389,7 @@ export function createUserApprovalRequestsRoutes(
const approvalId = String(req.params?.id || '').trim()
return approvalId ? `provider-decision:${approvalId}` : null
},
+ onBackendUnavailable: 'open',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcpOauth.ts b/control-api/src/routes/mcpOauth.ts
index c5ee97884..909d72b13 100644
--- a/control-api/src/routes/mcpOauth.ts
+++ b/control-api/src/routes/mcpOauth.ts
@@ -378,6 +378,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {
const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefined
return claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined
@@ -542,6 +543,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {
const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefined
return claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined
diff --git a/control-api/src/routes/recipeOauth.ts b/control-api/src/routes/recipeOauth.ts
index c933c4343..60fb8877d 100644
--- a/control-api/src/routes/recipeOauth.ts
+++ b/control-api/src/routes/recipeOauth.ts
@@ -81,6 +81,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth:${req.recipe.namespace}/${req.recipe.name}` : null,
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
const { namespace: recipeNamespace, name: recipeName } = req.recipe!
@@ -189,6 +190,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth-user:${req.recipe.namespace}/${req.recipe.name}` : null,
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
// [SEC-5] recipe identity ONLY from the broker token sub; userId is a
@@ -198,8 +200,10 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
const oauthClientId = (req.body ?? {}).oauthClientId
const userId = (req.body ?? {}).userId
if (
- typeof oauthClientId !== 'string' || oauthClientId.length === 0 ||
- typeof userId !== 'string' || userId.length === 0
+ typeof oauthClientId !== 'string' ||
+ oauthClientId.length === 0 ||
+ typeof userId !== 'string' ||
+ userId.length === 0
) {
return res.status(400).json({ error: 'invalid_request' })
}
@@ -220,7 +224,14 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
}
const result = await getAccessToken(
- { grantKind: 'user', recipeNamespace, recipeName, userId, oauthClientId, requireBackground: true },
+ {
+ grantKind: 'user',
+ recipeNamespace,
+ recipeName,
+ userId,
+ oauthClientId,
+ requireBackground: true,
+ },
{
db: { query: (text, values) => pool.query(text, values) },
recipeReader,
@@ -232,7 +243,14 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
const audit = (outcome: string) =>
req.log?.info(
- { event: 'recipe_oauth_user_token_issued', recipeNamespace, recipeName, userId, oauthClientId, outcome },
+ {
+ event: 'recipe_oauth_user_token_issued',
+ recipeNamespace,
+ recipeName,
+ userId,
+ oauthClientId,
+ outcome,
+ },
'recipe oauth user token issued'
)
@@ -254,13 +272,17 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
return res.status(400).json({ error: 'unknown_oauth_client' })
case 'unsupported_provider':
audit('unsupported_provider')
- return res.status(400).json({ error: 'unsupported_provider', provider: result.provider })
+ return res
+ .status(400)
+ .json({ error: 'unsupported_provider', provider: result.provider })
case 'secret_missing':
audit('integration_not_configured')
return res.status(503).json(integrationNotConfigured(oauthClientId, result.secret))
case 'refresh_failed':
audit('refresh_failed')
- return res.status(502).json({ error: 'refresh_failed', status: result.status, detail: result.detail })
+ return res
+ .status(502)
+ .json({ error: 'refresh_failed', status: result.status, detail: result.detail })
}
} catch (err) {
next(err)
@@ -283,6 +305,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth-users:${req.recipe.namespace}/${req.recipe.name}` : null,
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
// [SEC-6] recipe identity from the broker token sub; returns only THIS
@@ -314,7 +337,13 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
)
req.log?.info(
- { event: 'recipe_oauth_users_listed', recipeNamespace, recipeName, oauthClientId, count: users.length },
+ {
+ event: 'recipe_oauth_users_listed',
+ recipeNamespace,
+ recipeName,
+ oauthClientId,
+ count: users.length,
+ },
'recipe oauth users listed'
)
return res.status(200).json({ users })
diff --git a/control-api/src/routes/registry.ts b/control-api/src/routes/registry.ts
index b6b682bad..a698a5dd1 100644
--- a/control-api/src/routes/registry.ts
+++ b/control-api/src/routes/registry.ts
@@ -32,6 +32,7 @@ export function createRegistryRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `voucher:${sub}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req: UiAuthedRequest, res, next) => {
try {
diff --git a/control-api/src/routes/rpc-access/hosts.ts b/control-api/src/routes/rpc-access/hosts.ts
index 96d3c59fd..92b2b04be 100644
--- a/control-api/src/routes/rpc-access/hosts.ts
+++ b/control-api/src/routes/rpc-access/hosts.ts
@@ -61,6 +61,7 @@ export function createRpcAccessHostsRouter(gateway: K8sGateway): Router {
const hostRef = String(req.params?.hostRef || '').trim()
return hostRef ? `host-wake:${hostRef}` : null
},
+ onBackendUnavailable: 'open',
}),
async (req, res, next) => {
const hostRef = String(req.params.hostRef || '').trim()
diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.ts
index 18e5a2553..42a04e36f 100644
--- a/control-api/src/routes/workflows/shared/rateLimit.ts
+++ b/control-api/src/routes/workflows/shared/rateLimit.ts
@@ -152,6 +152,7 @@ function adminCodexReadRateLimit() {
bucketType: 'admin_codex_read',
maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),
+ onBackendUnavailable: 'open',
})
}
@@ -160,6 +161,7 @@ function adminCodexWriteRateLimit() {
bucketType: 'admin_codex_write',
maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),
+ onBackendUnavailable: 'open',
})
}
@@ -194,6 +196,7 @@ export function codexOAuthCallbackRateLimits() {
bucketType: 'codex_oauth_callback',
maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,
getBucketKey: codexOAuthCallbackBucketKey,
+ onBackendUnavailable: 'open',
}),
] as const
}
@@ -231,6 +234,7 @@ export function llmProviderAttemptAuthorizeRateLimits() {
bucketType: 'llm_provider_attempt_authorize',
maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,
getBucketKey: mcpHostAttemptRateLimitKey,
+ onBackendUnavailable: 'open',
}),
] as const
}
@@ -251,6 +255,7 @@ export function adminWorkflowTriggerRateLimit() {
bucketType: 'workflow_trigger',
maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,
getBucketKey: adminWorkflowTriggerRateLimitKey,
+ onBackendUnavailable: 'open',
})
}
@@ -268,6 +273,7 @@ function workflowTriggerRateLimitFor(getCredential: (req: Request) => string | n
if (!credential) return null
return hashedWorkflowTriggerBucket(credential)
},
+ onBackendUnavailable: 'open',
})
}
@@ -326,6 +332,7 @@ function workflowAdminReadRateLimit() {
bucketType: 'workflow_admin_read',
maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),
+ onBackendUnavailable: 'open',
})
}
@@ -334,6 +341,7 @@ function adminOutputsReadRateLimit() {
bucketType: 'admin_outputs_read',
maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),
+ onBackendUnavailable: 'open',
})
}
@@ -342,6 +350,7 @@ export function workflowGrantReadRateLimit() {
bucketType: 'workflow_grants_read',
maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),
+ onBackendUnavailable: 'open',
})
}
@@ -350,6 +359,7 @@ export function workflowGrantWriteRateLimit() {
bucketType: 'workflow_grants_write',
maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),
+ onBackendUnavailable: 'open',
})
}
diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.ts
new file mode 100644
index 000000000..e55b1424c
--- /dev/null
+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts
@@ -0,0 +1,118 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import type { RateLimitCheck } from '../src/services/rateLimiterService.js'
+
+const checkAndIncrement = vi.hoisted(() => vi.fn())
+const hits = vi.hoisted(() => ({ inc: vi.fn() }))
+
+vi.mock('../src/services/rateLimiterService.js', () => ({
+ checkAndIncrement,
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,
+}))
+vi.mock('../src/observability/metrics.js', () => ({ rateLimitHitsTotal: hits }))
+
+const { rateLimitMiddleware } = await import('../src/middleware/rateLimitMiddleware.js')
+
+const LIMIT = 5
+const resetMs = Date.now() + 30_000
+
+function available(count: number): RateLimitCheck {
+ return {
+ allowed: count <= LIMIT,
+ remaining: Math.max(0, LIMIT - count),
+ resetMs,
+ windowStartMs: resetMs - 60_000,
+ count,
+ backendAvailable: true,
+ }
+}
+
+/** What checkAndIncrement returns when it could not count the request. */
+function unavailable(): RateLimitCheck {
+ return {
+ allowed: true,
+ remaining: LIMIT,
+ resetMs,
+ windowStartMs: resetMs - 60_000,
+ count: 0,
+ backendAvailable: false,
+ }
+}
+
+function appWith(onBackendUnavailable: 'open' | 'closed') {
+ const handler = vi.fn((_req: express.Request, res: express.Response) => {
+ res.status(204).end()
+ })
+ const app = express()
+ app.get(
+ '/limited',
+ rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'unit:key',
+ onBackendUnavailable,
+ }),
+ handler
+ )
+ return { app, handler }
+}
+
+describe('rateLimitMiddleware backend-unavailable policy', () => {
+ beforeEach(() => {
+ checkAndIncrement.mockReset()
+ hits.inc.mockReset()
+ })
+
+ it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {
+ checkAndIncrement.mockResolvedValueOnce(unavailable())
+ const { app, handler } = appWith('closed')
+
+ const response = await request(app).get('/limited')
+
+ expect(response.status).toBe(503)
+ expect(response.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
+ expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['cache-control']).toBe('no-store')
+ // Liveness witness: the limiter was consulted with the caller's key.
+ expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)
+ expect(hits.inc).toHaveBeenCalledExactlyOnceWith(
+ { bucket_type: 'unit_bucket', result: 'unavailable' },
+ 1
+ )
+ expect(handler).not.toHaveBeenCalled()
+ })
+
+ it("'open' lets the uncounted request through, as before", async () => {
+ checkAndIncrement.mockResolvedValueOnce(unavailable())
+ const { app, handler } = appWith('open')
+
+ const response = await request(app).get('/limited')
+
+ expect(response.status).toBe(204)
+ expect(handler).toHaveBeenCalledTimes(1)
+ expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)
+ expect(hits.inc).toHaveBeenCalledExactlyOnceWith(
+ { bucket_type: 'unit_bucket', result: 'allowed' },
+ 1
+ )
+ })
+
+ it("'closed' still admits a counted request and still answers 429 over the limit", async () => {
+ checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))
+ const { app, handler } = appWith('closed')
+
+ const admitted = await request(app).get('/limited')
+ const denied = await request(app).get('/limited')
+
+ expect(admitted.status).toBe(204)
+ expect(admitted.headers['x-ratelimit-remaining']).toBe('4')
+ expect(denied.status).toBe(429)
+ expect(denied.body).toMatchObject({ error: 'Too Many Requests' })
+ expect(handler).toHaveBeenCalledTimes(1)
+ expect(hits.inc.mock.calls).toEqual([
+ [{ bucket_type: 'unit_bucket', result: 'allowed' }, 1],
+ [{ bucket_type: 'unit_bucket', result: 'denied' }, 1],
+ ])
+ })
+})
diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
index 82e18e0b1..f501ae2ca 100644
--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
@@ -65,7 +65,6 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
let mod: {
createApp: (gateway: unknown) => import('express').Express
config: typeof import('../src/config.js').config
- rootLogger: typeof import('../src/observability/logger.js').rootLogger
requestsTotal: typeof import('../src/observability/metrics.js').externalGfsRateLimitRequestsTotal
MockGateway: typeof import('./mockGateway.js').MockGateway
}
@@ -94,13 +93,11 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
const appMod = await import('../src/app.js')
const configMod = await import('../src/config.js')
- const loggerMod = await import('../src/observability/logger.js')
const metricsMod = await import('../src/observability/metrics.js')
const { MockGateway } = await import('./mockGateway.js')
mod = {
createApp: appMod.createApp as never,
config: configMod.config,
- rootLogger: loggerMod.rootLogger,
requestsTotal: metricsMod.externalGfsRateLimitRequestsTotal,
MockGateway,
}
@@ -229,4 +226,72 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
},
T3_TIMEOUT_MS
)
+
+ // Holding the pool (T3) cannot reach the grants rateLimitMiddleware: the
+ // pre-resolution Postgres bucket fails closed first. A trigger instead
+ // fails every limiter upsert whose key is not an external GFS Postgres
+ // bucket, so those keep counting and the grants bucket gets a real query
+ // error (plan addendum A2).
+ async function injectLimiterFault(): Promise<void> {
+ await corePool.query(`
+ CREATE FUNCTION t3b_limiter_fault() RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN
+ IF NEW.bucket_key NOT LIKE 'gfs-ext:%' THEN
+ RAISE EXCEPTION 't3b injected limiter fault';
+ END IF;
+ RETURN NEW;
+ END
+ $$`)
+ await corePool.query(`
+ CREATE TRIGGER t3b_limiter_fault BEFORE INSERT ON rate_limit_buckets
+ FOR EACH ROW EXECUTE FUNCTION t3b_limiter_fault()`)
+ }
+
+ async function removeLimiterFault(): Promise<void> {
+ await corePool.query('DROP TRIGGER IF EXISTS t3b_limiter_fault ON rate_limit_buckets')
+ await corePool.query('DROP FUNCTION IF EXISTS t3b_limiter_fault()')
+ }
+
+ it('T3b: the external grants limiter fails closed on a backend error while an open-policy route still passes', async () => {
+ const app = mod.createApp(new mod.MockGateway())
+ const userId = await seedUser()
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ const external = (req: request.Test) =>
+ req
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', JSON.stringify({ userId }))
+ .set('x-forwarded-for', '203.0.113.7')
+ await corePool.query('DELETE FROM rate_limit_buckets')
+
+ let grants: request.Response
+ let ack: request.Response
+ await injectLimiterFault()
+ try {
+ grants = await external(
+ request(app).get(`/api/v1/external/gfs/grants?drive=main&resourceId=${randomUUID()}`)
+ )
+ ack = await external(request(app).post(`/api/v1/external/notifications/${randomUUID()}/ack`))
+ } finally {
+ await removeLimiterFault()
+ }
+
+ // The 503 comes from rateLimitMiddleware, not from the external GFS
+ // Postgres buckets (whose body is gfs_rate_limit_unavailable).
+ expect(grants.status).toBe(503)
+ expect(grants.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
+ expect(grants.headers['retry-after']).toBe('2')
+ expect(grants.headers['cache-control']).toBe('no-store')
+ // Witness for the open policy under the same fault: the ack route's
+ // limiter could not count either, and its handler still answered.
+ expect(ack.status).toBe(404)
+ expect(ack.body).toEqual({ error: 'notification_not_found' })
+
+ // Witness that the fault is scoped as intended: the external GFS Postgres
+ // buckets counted, and nothing else reached the ledger.
+ const keys = await ledgerKeys()
+ expect(keys.length).toBeGreaterThan(0)
+ expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)
+ }, 30_000)
})