Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(control-api): count requests in process memory while the Postgres limiter is unavailable
Alfredo Lopeza9964ddc8ac7
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 9479475ab..deae46cb8 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -308,6 +308,7 @@ jobs:services.llmCatalogSync.realPostgres.integration \routes.externalGfsRateLimitStress.realPostgres.integration \routes.externalGfsRateLimitFailClosed.realPostgres.integration \+ routes.rateLimitProcessMemory.realPostgres.integration \services.rateLimiterPoolSaturation.realPostgres.integrationdogrep -Fq "$suite" "$normalized_log" || {diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 8090d616a..f1b260b97 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.617",+ "version": "0.1.618","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.617",+ "version": "0.1.618","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 9e658aedb..2bee1119c 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.617",+ "version": "0.1.618","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.tsindex 58f9d9ebd..57bd92721 100644--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts@@ -32,7 +32,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {// ENV-tunable platform limit (issue #348): CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN.maxPerMinute: config.pluginSdkRequestBucketRlPerMin,getBucketKey: pluginWorkloadSdkRequestBucketKey,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -46,7 +46,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {if (!claims) return 'plugin_workload_sdk_internal:unauthenticated'return `plugin_workload_sdk_internal:${claims.iss}:${claims.sub}`},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -64,6 +64,6 @@ export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {const sub = (req as UiAuthedRequest).adminAuth?.subreturn `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}diff --git a/control-api/src/middleware/processMemoryRateLimiter.ts b/control-api/src/middleware/processMemoryRateLimiter.tsnew file mode 100644index 000000000..86c69b992--- /dev/null+++ b/control-api/src/middleware/processMemoryRateLimiter.ts@@ -0,0 +1,62 @@+import { MemoryStore, type Options } from 'express-rate-limit'++/** Distinct keys one counter accepts per window before it refuses new ones. */+export const PROCESS_MEMORY_MAX_TRACKED_KEYS = 100_000++const WINDOW_MS = 60_000++export type ProcessMemoryDecision =+ | { outcome: 'allowed' | 'denied'; totalHits: number; resetMs: number }+ | { outcome: 'key_cap_reached' }++/**+ * A per-minute counter held in this process's memory. `rateLimitMiddleware`+ * uses it only while the Postgres limiter cannot count a request+ * (`onBackendUnavailable: 'process-memory'`).+ *+ * Each key gets a fixed 60 s window that starts at its first hit+ * (express-rate-limit's `MemoryStore`). A request is allowed while the key's+ * hit count in its window is at most `maxPerMinute`.+ *+ * `MemoryStore` has no key count, so this class records the distinct keys it+ * has seen in its own 60 s window, and refuses a new key once that set holds+ * `maxTrackedKeys`. The store keeps a key until its cleanup timer has run+ * twice without a hit on it, so it can hold a few times `maxTrackedKeys`+ * entries at once; the cap bounds how fast that grows.+ */+export class ProcessMemoryRateLimiter {+ private readonly store = new MemoryStore()+ private readonly keysThisWindow = new Set<string>()+ private windowStartMs = Date.now()++ constructor(+ private readonly maxPerMinute: number,+ private readonly maxTrackedKeys: number = PROCESS_MEMORY_MAX_TRACKED_KEYS+ ) {+ // MemoryStore.init reads only `windowMs` from the options. It starts an+ // unref'd cleanup timer, so an idle counter does not keep the process up.+ this.store.init({ windowMs: WINDOW_MS } as Options)+ }++ async hit(key: string): Promise<ProcessMemoryDecision> {+ const now = Date.now()+ if (now - this.windowStartMs >= WINDOW_MS) {+ this.keysThisWindow.clear()+ this.windowStartMs = now+ }+ if (!this.keysThisWindow.has(key)) {+ if (this.keysThisWindow.size >= this.maxTrackedKeys) return { outcome: 'key_cap_reached' }+ this.keysThisWindow.add(key)+ }++ const { totalHits, resetTime } = await this.store.increment(key)+ if (resetTime === undefined) {+ throw new Error('express-rate-limit MemoryStore returned a hit without a reset time')+ }+ return {+ outcome: totalHits <= this.maxPerMinute ? 'allowed' : 'denied',+ totalHits,+ resetMs: resetTime.getTime(),+ }+ }+}diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.tsindex 63f01c186..0bcd39d94 100644--- a/control-api/src/middleware/rateLimitMiddleware.ts+++ b/control-api/src/middleware/rateLimitMiddleware.ts@@ -1,10 +1,12 @@import { NextFunction, Request, Response } from 'express'import { createHash } from 'node:crypto'+import { LogThrottle } from '../observability/logThrottle.js'import { rateLimitHitsTotal } from '../observability/metrics.js'import {RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,checkAndIncrement,} from '../services/rateLimiterService.js'+import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'/*** Factory for a per-request rate limit middleware backed by a PG token bucket.@@ -15,6 +17,7 @@ import {* bucketType: "recipe",* maxPerMinute: 10,* getBucketKey: (req) => mcpHostRateLimitBucketKey('recipe', req.mcpHostJwt),+ * onBackendUnavailable: 'process-memory',* }),* handler,* );@@ -22,21 +25,90 @@ import {* The middleware:* - Returns 429 with `Retry-After` (seconds until window end) when the limit* is exceeded.- * - Emits `rate_limit_hits_total{bucket_type, result}` on every call.+ * - Emits `rate_limit_hits_total{bucket_type, result}` on every call, with+ * `result` one of `allowed`, `denied`, `unavailable`, `fallback_allowed`,+ * `fallback_denied`.* - If `getBucketKey` returns `null` (e.g. unable to derive key), the* request is *allowed* but not counted — fail-open. Callers that need* strict enforcement should return a sentinel key instead.* - When the limiter backend cannot count the request- * (`backendAvailable: false`), `onBackendUnavailable` decides: `'open'`- * lets it through uncounted, `'closed'` answers 503 with `Retry-After`.- * The option is required so every caller states that choice.+ * (`backendAvailable: false`: the limiter pool is saturated, Postgres+ * returned an error, or the upsert returned no row), `onBackendUnavailable`+ * decides. The option is required so every caller states that choice.+ * - `'closed'` answers 503 `rate_limit_unavailable` with `Retry-After`.+ * - `'process-memory'` counts the request in this middleware instance's+ * own in-memory counter: the same `maxPerMinute`, in a 60 s window per+ * key. Over the limit it answers 429 like a Postgres denial, and the+ * warn line carries `source: 'process-memory'`. The in-memory counter+ * is used only for requests Postgres could not count; the two counts+ * are never added together. It tracks at most 100 000 keys per 60 s;+ * a new key beyond that answers 503 as `'closed'` does. The counter+ * lives in one process: control-api runs one replica today, so the+ * in-memory limit holds for the whole service only while that is+ * true. With N replicas a caller can get N × `maxPerMinute` requests+ * per minute while Postgres is unavailable.*/export function rateLimitMiddleware(opts: {bucketType: stringmaxPerMinute: numbergetBucketKey: (req: Request) => string | null- onBackendUnavailable: 'open' | 'closed'+ onBackendUnavailable: 'process-memory' | 'closed'}) {+ // While the backend is down every request on this bucket fails the same way,+ // so the warn line is written once per minute; the counter records each one.+ const unavailableLogThrottle = new LogThrottle(60_000)+ const processMemory =+ opts.onBackendUnavailable === 'process-memory'+ ? new ProcessMemoryRateLimiter(opts.maxPerMinute)+ : null++ function answerUnavailable(req: Request, res: Response): void {+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)+ const suppressed = unavailableLogThrottle.admit(opts.bucketType)+ if (req.log && suppressed !== undefined) {+ req.log.warn(+ { event: 'rate_limit_unavailable', bucketType: opts.bucketType, suppressed },+ 'rate limit backend unavailable'+ )+ }+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))+ res.setHeader('Cache-Control', 'no-store')+ res.status(503).json({+ error: 'rate_limit_unavailable',+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ })+ }++ function answerDenied(+ req: Request,+ res: Response,+ key: string,+ denial: { count: number; resetMs: number; source?: 'process-memory' }+ ): void {+ const retryAfterSec = Math.max(1, Math.ceil((denial.resetMs - Date.now()) / 1000))+ res.setHeader('Retry-After', String(retryAfterSec))+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))+ res.setHeader('X-RateLimit-Remaining', '0')+ res.setHeader('X-RateLimit-Reset', String(Math.floor(denial.resetMs / 1000)))+ if (req.log) {+ req.log.warn(+ {+ event: 'rate_limit_denied',+ bucketType: opts.bucketType,+ // Keys carry user ids (e.g. desktopUserId). The SHA-256 is a+ // correlation id, not anonymization: it is unsalted, so a+ // low-entropy key (an IP, an email) can be recovered by guessing.+ hashedKey: createHash('sha256').update(key).digest('hex'),+ count: denial.count,+ maxPerMinute: opts.maxPerMinute,+ ...(denial.source === undefined ? {} : { source: denial.source }),+ },+ 'rate limit exceeded'+ )+ }+ res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })+ }+return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {void (async () => {try {@@ -49,43 +121,35 @@ export function rateLimitMiddleware(opts: {}const result = await checkAndIncrement(key, opts.maxPerMinute)- if (!result.backendAvailable && opts.onBackendUnavailable === 'closed') {- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)- if (req.log) {- req.log.warn(- { event: 'rate_limit_unavailable', bucketType: opts.bucketType },- 'rate limit backend unavailable'- )+ if (!result.backendAvailable) {+ if (processMemory === null) {+ answerUnavailable(req, res)+ return+ }+ const decision = await processMemory.hit(key)+ if (decision.outcome === 'key_cap_reached') {+ answerUnavailable(req, res)+ return}- res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))- res.setHeader('Cache-Control', 'no-store')- res.status(503).json({- error: 'rate_limit_unavailable',- retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,- })+ if (decision.outcome === 'denied') {+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)+ answerDenied(req, res, key, {+ count: decision.totalHits,+ resetMs: decision.resetMs,+ source: 'process-memory',+ })+ return+ }+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))+ res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))+ res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))+ next()return}if (!result.allowed) {rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)- const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))- res.setHeader('Retry-After', String(retryAfterSec))- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', '0')- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))- if (req.log) {- req.log.warn(- {- event: 'rate_limit_denied',- bucketType: opts.bucketType,- // Keys carry user ids (e.g. desktopUserId); log their SHA-256.- hashedKey: createHash('sha256').update(key).digest('hex'),- count: result.count,- maxPerMinute: opts.maxPerMinute,- },- 'rate limit exceeded'- )- }- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })+ answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })return}diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.tsindex e8b976352..c40efc41b 100644--- a/control-api/src/routes/admin/auth.ts+++ b/control-api/src/routes/admin/auth.ts@@ -1,5 +1,6 @@-import { Router } from 'express'+import { type Request, type RequestHandler, Router } from 'express'import bcrypt from 'bcryptjs'+import { ipKeyGenerator } from 'express-rate-limit'import { config } from '../../config.js'import { pool } from '../../db.js'import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'@@ -50,8 +51,26 @@ const ADMIN_PASSWORD_MAX_LENGTH = 256const logger = rootLogger.child({ module: 'admin-auth' })-function publicAdminTokenRateLimit(routeName: string) {- return rateLimitMiddleware({+function remoteAddressOf(req: Request): string {+ return req.ip || req.socket.remoteAddress || 'unknown'+}++/**+ * Two buckets per public token route, applied in this order:+ *+ * 1. Source IP + submitted value (email, login or token prefix), at+ * `adminPublicTokenRlPerMin`: one person retrying the same value.+ * 2. Source IP alone (IPv6 grouped by /56), at `adminPublicTokenIpRlPerMin`:+ * the ceiling on rotating values from one source, which the first bucket+ * cannot see because every new value gets a fresh key.+ *+ * Many admins can share one public IP (a corporate VPN egress, an office+ * NAT), so the second ceiling is several times the first. A request the first+ * bucket refuses never reaches the second, so one person's retry loop costs+ * the shared IP at most `adminPublicTokenRlPerMin` per minute.+ */+function publicAdminTokenRateLimits(routeName: string): [RequestHandler, RequestHandler] {+ const perValue = rateLimitMiddleware({bucketType: `control_admin_public_${routeName}`,maxPerMinute: config.adminPublicTokenRlPerMin,getBucketKey: req => {@@ -60,11 +79,18 @@ function publicAdminTokenRateLimit(routeName: string) {.trim().toLowerCase()const login = String(req.body?.username || req.body?.login || '').trim()- const remote = req.ip || req.socket.remoteAddress || 'unknown'- return `control-admin-public:${routeName}:${remote}:${email || login || token.slice(0, 48) || 'missing'}`+ return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${email || login || token.slice(0, 48) || 'missing'}`},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',+ })+ const perIp = rateLimitMiddleware({+ bucketType: `control_admin_public_ip_${routeName}`,+ maxPerMinute: config.adminPublicTokenIpRlPerMin,+ getBucketKey: req =>+ `control-admin-public-ip:${routeName}:${ipKeyGenerator(remoteAddressOf(req))}`,+ onBackendUnavailable: 'process-memory',})+ return [perValue, perIp]}export function createAdminAuthRouter(): Router {@@ -240,7 +266,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/password-reset/request',- publicAdminTokenRateLimit('password_reset_request'),+ ...publicAdminTokenRateLimits('password_reset_request'),async (req, res, next) => {try {const username = String(req.body?.username || '').trim()@@ -276,7 +302,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/password-reset/validate',- publicAdminTokenRateLimit('password_reset_validate'),+ ...publicAdminTokenRateLimits('password_reset_validate'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()@@ -313,7 +339,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/password-reset/complete',- publicAdminTokenRateLimit('password_reset_complete'),+ ...publicAdminTokenRateLimits('password_reset_complete'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()@@ -370,7 +396,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/control-admin-invitations/validate',- publicAdminTokenRateLimit('invitation_validate'),+ ...publicAdminTokenRateLimits('invitation_validate'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()@@ -409,7 +435,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/control-admin-invitations/complete',- publicAdminTokenRateLimit('invitation_complete'),+ ...publicAdminTokenRateLimits('invitation_complete'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()@@ -501,7 +527,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/control-admin-email-confirmations/validate',- publicAdminTokenRateLimit('email_confirmation_validate'),+ ...publicAdminTokenRateLimits('email_confirmation_validate'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()@@ -541,7 +567,7 @@ export function createAdminAuthRouter(): Router {router.post('/admin/auth/control-admin-email-confirmations/complete',- publicAdminTokenRateLimit('email_confirmation_complete'),+ ...publicAdminTokenRateLimits('email_confirmation_complete'),async (req, res, next) => {try {const token = String(req.body?.token || '').trim()diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex eee584e8b..0b098f0fe 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -3166,7 +3166,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orgkeys:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})// Guards ⓪①② — returns null after sending an error response.@@ -3288,7 +3288,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orggrants:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})// Resolve the caller's own org for a self-service op, or send the terminal errordiff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.tsindex 25357ad1f..35811c028 100644--- a/control-api/src/routes/admin/registryConnect.ts+++ b/control-api/src/routes/admin/registryConnect.ts@@ -223,7 +223,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_status:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {@@ -334,7 +334,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_request:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {@@ -593,7 +593,7 @@ export function createRegistryConnectRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_recover:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.tsindex e50c500ad..e8dfb0aa8 100644--- a/control-api/src/routes/external/auth.ts+++ b/control-api/src/routes/external/auth.ts@@ -129,7 +129,7 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {.toLowerCase()return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/decisions.routes.ts b/control-api/src/routes/external/decisions.routes.tsindex 4967cd6c8..d72e50533 100644--- a/control-api/src/routes/external/decisions.routes.ts+++ b/control-api/src/routes/external/decisions.routes.ts@@ -63,7 +63,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req, res, next) => {void (async () => {@@ -103,7 +103,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),requireApprovalDecisionAccess(),(req, res, next) => {diff --git a/control-api/src/routes/external/invitations.ts b/control-api/src/routes/external/invitations.tsindex a7e8c0c75..bacba9eba 100644--- a/control-api/src/routes/external/invitations.ts+++ b/control-api/src/routes/external/invitations.ts@@ -46,7 +46,7 @@ export function createExternalInvitationsRouter(): Router {bucketType: 'external_invitation_lookup',maxPerMinute: 30,getBucketKey: req => invitationLookupIpKey(req),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {@@ -80,7 +80,7 @@ export function createExternalInvitationsRouter(): Router {bucketType: 'external_invitation_password_token',maxPerMinute: 10,getBucketKey: req => invitationLookupIpKey(req),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/external/notifications.routes.ts b/control-api/src/routes/external/notifications.routes.tsindex 23f25adfe..0ab0bba9e 100644--- a/control-api/src/routes/external/notifications.routes.ts+++ b/control-api/src/routes/external/notifications.routes.ts@@ -111,7 +111,7 @@ export function createExternalNotificationsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}:notifications` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req: Request, res: Response) => {void (async () => {@@ -345,7 +345,7 @@ export function createExternalNotificationsRouter(): Router {const uid = extReq.externalAuth?.userIdreturn uid ? `user:${uid}:notification-ack` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),asyncHandler(async (req: Request, res: Response) => {const extReq = req as ExternalAuthedRequestdiff --git a/control-api/src/routes/external/users.ts b/control-api/src/routes/external/users.tsindex 5378b7ca8..7f07f440b 100644--- a/control-api/src/routes/external/users.ts+++ b/control-api/src/routes/external/users.ts@@ -107,7 +107,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {const userId = (req as ExternalAuthedRequest).externalAuth?.userIdreturn userId ? `team_directory:${userId}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {@@ -285,7 +285,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {const userId = (req as ExternalAuthedRequest).externalAuth?.userIdreturn userId ? `profile_password_update:${userId}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {try {diff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.tsindex 007aa15ef..0faa9c705 100644--- a/control-api/src/routes/gfs/grants.ts+++ b/control-api/src/routes/gfs/grants.ts@@ -896,7 +896,7 @@ export function registerGfsGrantRoutes(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})router.get('/gfs/grants', requireAuthForControlUI, grantsRateLimit, asyncHandler(handleGrantRead))router.put(diff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.tsindex 4586ea55f..16daaf772 100644--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts@@ -69,7 +69,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants-legacy:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})router.get('/gfs/grants/legacy-standalone',diff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.tsindex e16c98697..0bdd0c325 100644--- a/control-api/src/routes/gfs/shares.ts+++ b/control-api/src/routes/gfs/shares.ts@@ -45,7 +45,7 @@ export function registerGfsShareRoutes(router: Router): void {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsshares:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})router.get('/gfs/shares', requireAuthForControlUI, sharesRateLimit, asyncHandler(handleShareRead))router.post(diff --git a/control-api/src/routes/internal/oauth.ts b/control-api/src/routes/internal/oauth.tsindex f6b247c96..59e2f87e3 100644--- a/control-api/src/routes/internal/oauth.ts+++ b/control-api/src/routes/internal/oauth.ts@@ -77,7 +77,7 @@ export function createInternalOAuthRouter(gateway: K8sGateway): Router {: (req.internalService?.name ?? 'unknown')return `mcp-oauth-internal:${uid}`},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})const recipeReader: RecipeReader = {diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.tsindex 21c56d2f6..931f43628 100644--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts@@ -82,7 +82,7 @@ const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({bucketType: 'plugin_workload_sdk_credential',maxPerMinute: 120,getBucketKey: pluginWorkloadSdkCredentialBucketKey,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})function requirePluginWorkloadSdkScope(req: Request, res: Response, next: NextFunction): void {diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.tsindex 80d3f72f5..dddf304cb 100644--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts@@ -31,7 +31,7 @@ export function createMcpHostRefreshRoutes(): Router {const key = getMcpHostRefreshRateLimitKey(token)return key ? `refresh:${key}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts b/control-api/src/routes/mcp-host/renewal/reissue.routes.tsindex c84614eaf..d8d63a46d 100644--- a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts+++ b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts@@ -37,7 +37,7 @@ export function createMcpHostReissueRoutes(): Router {const key = getMcpHostExpiredRefreshRateLimitKey(token)return key ? `reissue:${key}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.tsindex 763dc71e2..948d05d55 100644--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts@@ -491,7 +491,7 @@ export function createUserApprovalRequestsRoutes(if (!auth) return nullreturn mcpHostRateLimitBucketKey('recipe', auth)},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req, res, next) => {void (async () => {@@ -1389,7 +1389,7 @@ export function createUserApprovalRequestsRoutes(const approvalId = String(req.params?.id || '').trim()return approvalId ? `provider-decision:${approvalId}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),(req, res, next) => {void (async () => {diff --git a/control-api/src/routes/mcpOauth.ts b/control-api/src/routes/mcpOauth.tsindex 909d72b13..d7368b0e6 100644--- a/control-api/src/routes/mcpOauth.ts+++ b/control-api/src/routes/mcpOauth.ts@@ -378,7 +378,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefinedreturn claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined@@ -543,7 +543,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefinedreturn claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {const claims = res.locals.mcpHostControl as McpHostControlClaims | undefineddiff --git a/control-api/src/routes/recipeOauth.ts b/control-api/src/routes/recipeOauth.tsindex 60fb8877d..dae4f1baa 100644--- a/control-api/src/routes/recipeOauth.ts+++ b/control-api/src/routes/recipeOauth.ts@@ -81,7 +81,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth:${req.recipe.namespace}/${req.recipe.name}` : null,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {const { namespace: recipeNamespace, name: recipeName } = req.recipe!@@ -190,7 +190,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth-user:${req.recipe.namespace}/${req.recipe.name}` : null,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {// [SEC-5] recipe identity ONLY from the broker token sub; userId is a@@ -305,7 +305,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {maxPerMinute: config.oauthBrokerRlPerMin,getBucketKey: req =>req.recipe ? `recipe-oauth-users:${req.recipe.namespace}/${req.recipe.name}` : null,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {// [SEC-6] recipe identity from the broker token sub; returns only THISdiff --git a/control-api/src/routes/registry.ts b/control-api/src/routes/registry.tsindex a698a5dd1..f64ea28cb 100644--- a/control-api/src/routes/registry.ts+++ b/control-api/src/routes/registry.ts@@ -32,7 +32,7 @@ export function createRegistryRouter(): Router {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `voucher:${sub}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req: UiAuthedRequest, res, next) => {try {diff --git a/control-api/src/routes/rpc-access/hosts.ts b/control-api/src/routes/rpc-access/hosts.tsindex 92b2b04be..e355357c0 100644--- a/control-api/src/routes/rpc-access/hosts.ts+++ b/control-api/src/routes/rpc-access/hosts.ts@@ -61,7 +61,7 @@ export function createRpcAccessHostsRouter(gateway: K8sGateway): Router {const hostRef = String(req.params?.hostRef || '').trim()return hostRef ? `host-wake:${hostRef}` : null},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),async (req, res, next) => {const hostRef = String(req.params.hostRef || '').trim()diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.tsindex 42a04e36f..8480d1980 100644--- a/control-api/src/routes/workflows/shared/rateLimit.ts+++ b/control-api/src/routes/workflows/shared/rateLimit.ts@@ -152,7 +152,7 @@ function adminCodexReadRateLimit() {bucketType: 'admin_codex_read',maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -161,7 +161,7 @@ function adminCodexWriteRateLimit() {bucketType: 'admin_codex_write',maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -196,7 +196,7 @@ export function codexOAuthCallbackRateLimits() {bucketType: 'codex_oauth_callback',maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,getBucketKey: codexOAuthCallbackBucketKey,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),] as const}@@ -234,7 +234,7 @@ export function llmProviderAttemptAuthorizeRateLimits() {bucketType: 'llm_provider_attempt_authorize',maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,getBucketKey: mcpHostAttemptRateLimitKey,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',}),] as const}@@ -255,7 +255,7 @@ export function adminWorkflowTriggerRateLimit() {bucketType: 'workflow_trigger',maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,getBucketKey: adminWorkflowTriggerRateLimitKey,- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -273,7 +273,7 @@ function workflowTriggerRateLimitFor(getCredential: (req: Request) => string | nif (!credential) return nullreturn hashedWorkflowTriggerBucket(credential)},- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -332,7 +332,7 @@ function workflowAdminReadRateLimit() {bucketType: 'workflow_admin_read',maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -341,7 +341,7 @@ function adminOutputsReadRateLimit() {bucketType: 'admin_outputs_read',maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -350,7 +350,7 @@ export function workflowGrantReadRateLimit() {bucketType: 'workflow_grants_read',maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}@@ -359,7 +359,7 @@ export function workflowGrantWriteRateLimit() {bucketType: 'workflow_grants_write',maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),- onBackendUnavailable: 'open',+ onBackendUnavailable: 'process-memory',})}diff --git a/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts b/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.tsindex e234d4568..d5a3fb35f 100644--- a/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts+++ b/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts@@ -45,6 +45,7 @@ const RESOURCE_ID = '20000000-0000-4000-8000-000000000001'function allowed(remaining = 29) {return {allowed: true,+ backendAvailable: true,remaining,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -55,6 +56,7 @@ function allowed(remaining = 29) {function denied() {return {allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30_000,windowStartMs: Date.now(),diff --git a/control-api/test/middleware.processMemoryRateLimiter.test.ts b/control-api/test/middleware.processMemoryRateLimiter.test.tsnew file mode 100644index 000000000..0cd45b10c--- /dev/null+++ b/control-api/test/middleware.processMemoryRateLimiter.test.ts@@ -0,0 +1,60 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import {+ PROCESS_MEMORY_MAX_TRACKED_KEYS,+ ProcessMemoryRateLimiter,+} from '../src/middleware/processMemoryRateLimiter.js'++const START_MS = 1_800_000_000_000++describe('ProcessMemoryRateLimiter', () => {+ afterEach(() => {+ vi.useRealTimers()+ })++ it('allows maxPerMinute hits per key in a 60 s window that starts at the first hit', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(START_MS)+ const limiter = new ProcessMemoryRateLimiter(2)++ const first = await limiter.hit('a')+ vi.setSystemTime(START_MS + 10_000)+ const second = await limiter.hit('a')+ const third = await limiter.hit('a')+ const otherKey = await limiter.hit('b')+ vi.setSystemTime(START_MS + 60_000)+ const nextWindow = await limiter.hit('a')++ expect(first).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 60_000 })+ expect(second).toEqual({ outcome: 'allowed', totalHits: 2, resetMs: START_MS + 60_000 })+ expect(third).toEqual({ outcome: 'denied', totalHits: 3, resetMs: START_MS + 60_000 })+ expect(otherKey).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 70_000 })+ expect(nextWindow).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 120_000 })+ })++ it('refuses a new key once maxTrackedKeys keys are tracked in the window, and accepts new keys in the next window', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(START_MS)+ const limiter = new ProcessMemoryRateLimiter(5, 2)++ expect((await limiter.hit('a')).outcome).toBe('allowed')+ expect((await limiter.hit('b')).outcome).toBe('allowed')+ expect(await limiter.hit('c')).toEqual({ outcome: 'key_cap_reached' })+ // Witness: a tracked key is still counted while the cap refuses new ones.+ expect(await limiter.hit('a')).toMatchObject({ outcome: 'allowed', totalHits: 2 })++ vi.setSystemTime(START_MS + 60_000)+ expect(await limiter.hit('c')).toMatchObject({ outcome: 'allowed', totalHits: 1 })+ })++ it('tracks at most 100 000 keys by default', async () => {+ expect(PROCESS_MEMORY_MAX_TRACKED_KEYS).toBe(100_000)+ const limiter = new ProcessMemoryRateLimiter(1)+ for (let n = 0; n < 100_000; n += 1) {+ const decision = await limiter.hit(`key:${n}`)+ if (decision.outcome !== 'allowed') throw new Error(`key:${n}: ${decision.outcome}`)+ }+ expect(await limiter.hit('key:100000')).toEqual({ outcome: 'key_cap_reached' })+ // Witness: the counter was live for the keys under the cap.+ expect(await limiter.hit('key:0')).toMatchObject({ outcome: 'denied', totalHits: 2 })+ })+})diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.tsindex 58c936f5a..03f1b68e6 100644--- a/control-api/test/middleware.rateLimitMiddleware.test.ts+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts@@ -1,5 +1,6 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { MemoryStore } from 'express-rate-limit'import { createHash } from 'node:crypto'import request from 'supertest'import type { RateLimitCheck } from '../src/services/rateLimiterService.js'@@ -41,13 +42,30 @@ function unavailable(): RateLimitCheck {}}-function appWith(onBackendUnavailable: 'open' | 'closed') {+// Compile-time check only. control-api's tsconfig covers src/, so vitest does+// not typecheck this file; run tsc over it to verify the expectation below.+export function buildWithRemovedOpenMode() {+ return rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'unit:key',+ // @ts-expect-error 'open' is no longer a mode+ onBackendUnavailable: 'open',+ })+}++function appWith(onBackendUnavailable: 'process-memory' | 'closed') {const handler = vi.fn((_req: express.Request, res: express.Response) => {res.status(204).end()})+ const warn = vi.fn()const app = express()app.get('/limited',+ (req, _res, next) => {+ ;(req as unknown as { log: { warn: typeof warn } }).log = { warn }+ next()+ },rateLimitMiddleware({bucketType: 'unit_bucket',maxPerMinute: LIMIT,@@ -56,7 +74,7 @@ function appWith(onBackendUnavailable: 'open' | 'closed') {}),handler)- return { app, handler }+ return { app, handler, warn }}describe('rateLimitMiddleware backend-unavailable policy', () => {@@ -67,11 +85,15 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {checkAndIncrement.mockResolvedValueOnce(unavailable())- const { app, handler } = appWith('closed')+ const { app, handler, warn } = appWith('closed')const response = await request(app).get('/limited')expect(response.status).toBe(503)+ expect(warn).toHaveBeenCalledExactlyOnceWith(+ { event: 'rate_limit_unavailable', bucketType: 'unit_bucket', suppressed: 0 },+ 'rate limit backend unavailable'+ )expect(response.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })expect(response.headers['retry-after']).toBe('2')expect(response.headers['cache-control']).toBe('no-store')@@ -84,21 +106,175 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {expect(handler).not.toHaveBeenCalled()})- it("'open' lets the uncounted request through, as before", async () => {- checkAndIncrement.mockResolvedValueOnce(unavailable())- const { app, handler } = appWith('open')+ it("'closed' writes one warn line per minute while the backend is down, and counts every 503", async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ vi.setSystemTime(1_800_000_000_000)+ checkAndIncrement.mockResolvedValue(unavailable())+ const { app, handler, warn } = appWith('closed')- const response = await request(app).get('/limited')+ const statuses: number[] = []+ for (let n = 0; n < 50; n += 1) statuses.push((await request(app).get('/limited')).status)- expect(response.status).toBe(204)- expect(handler).toHaveBeenCalledTimes(1)- expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)- expect(hits.inc).toHaveBeenCalledExactlyOnceWith(- { bucket_type: 'unit_bucket', result: 'allowed' },- 1+ // Witness: all 50 reached the limiter and were refused.+ expect(statuses.filter(status => status === 503)).toHaveLength(50)+ expect(checkAndIncrement).toHaveBeenCalledTimes(50)+ expect(hits.inc).toHaveBeenCalledTimes(50)+ expect(warn).toHaveBeenCalledTimes(1)++ vi.setSystemTime(1_800_000_060_000)+ expect((await request(app).get('/limited')).status).toBe(503)+ expect(warn).toHaveBeenCalledTimes(2)+ expect(warn.mock.calls[1]![0]).toMatchObject({+ event: 'rate_limit_unavailable',+ suppressed: 49,+ })+ expect(handler).not.toHaveBeenCalled()+ } finally {+ vi.useRealTimers()+ }+ })++ it("'process-memory' counts in memory while the backend cannot count, and answers 429 over the limit", async () => {+ checkAndIncrement.mockResolvedValue(unavailable())+ const { app, handler, warn } = appWith('process-memory')++ const responses: request.Response[] = []+ for (let n = 0; n <= LIMIT; n += 1) responses.push(await request(app).get('/limited'))++ // Witness: every request reached the Postgres limiter first.+ expect(checkAndIncrement).toHaveBeenCalledTimes(LIMIT + 1)+ expect(responses.slice(0, LIMIT).map(response => response.status)).toEqual(+ Array.from({ length: LIMIT }, () => 204))+ expect(+ responses.slice(0, LIMIT).map(response => response.headers['x-ratelimit-remaining'])+ ).toEqual(['4', '3', '2', '1', '0'])+ expect(handler).toHaveBeenCalledTimes(LIMIT)++ const denied = responses[LIMIT]!+ expect(denied.status).toBe(429)+ const retryAfterSeconds = Number(denied.headers['retry-after'])+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)+ expect(retryAfterSeconds).toBeLessThanOrEqual(60)+ expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })+ expect(denied.headers['x-ratelimit-limit']).toBe(String(LIMIT))+ expect(denied.headers['x-ratelimit-remaining']).toBe('0')+ expect(Number(denied.headers['x-ratelimit-reset'])).toBeGreaterThan(Date.now() / 1000)++ expect(warn).toHaveBeenCalledExactlyOnceWith(+ {+ event: 'rate_limit_denied',+ bucketType: 'unit_bucket',+ hashedKey: createHash('sha256').update('unit:key').digest('hex'),+ count: LIMIT + 1,+ maxPerMinute: LIMIT,+ source: 'process-memory',+ },+ 'rate limit exceeded'+ )+ expect(hits.inc.mock.calls).toEqual([+ ...Array.from({ length: LIMIT }, () => [+ { bucket_type: 'unit_bucket', result: 'fallback_allowed' },+ 1,+ ]),+ [{ bucket_type: 'unit_bucket', result: 'fallback_denied' }, 1],+ ])+ })++ it("'process-memory' leaves the decision to Postgres again once the backend can count", async () => {+ const increment = vi.spyOn(MemoryStore.prototype, 'increment')+ try {+ for (let n = 0; n <= LIMIT; n += 1) checkAndIncrement.mockResolvedValueOnce(unavailable())+ checkAndIncrement.mockResolvedValueOnce(available(1))+ const { app, handler } = appWith('process-memory')++ const statuses: number[] = []+ for (let n = 0; n <= LIMIT + 1; n += 1)+ statuses.push((await request(app).get('/limited')).status)++ // The in-memory counter is exhausted, yet the counted request passes:+ // Postgres decided it, and the in-memory store was not consulted.+ expect(statuses).toEqual([...Array.from({ length: LIMIT }, () => 204), 429, 204])+ expect(increment).toHaveBeenCalledTimes(LIMIT + 1)+ // Liveness witness: the last request reached the Postgres limiter and the handler.+ expect(checkAndIncrement).toHaveBeenCalledTimes(LIMIT + 2)+ expect(handler).toHaveBeenCalledTimes(LIMIT + 1)+ expect(hits.inc.mock.calls.at(-1)).toEqual([+ { bucket_type: 'unit_bucket', result: 'allowed' },+ 1,+ ])+ } finally {+ increment.mockRestore()+ }})+ it("'process-memory' answers 503 for a new key once 100 000 keys are tracked, and still counts tracked keys", async () => {+ checkAndIncrement.mockResolvedValue(unavailable())+ const middleware = rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: req => req.headers['x-key'] as string,+ onBackendUnavailable: 'process-memory',+ })+ // Driven without HTTP: 100 001 supertest round trips would take minutes.+ type Outcome = {+ next: boolean+ status?: number+ body?: unknown+ headers: Record<string, string>+ }+ const call = (key: string) =>+ new Promise<Outcome>((resolve, reject) => {+ const headers: Record<string, string> = {}+ const res = {+ statusCode: 200,+ setHeader(name: string, value: string) {+ headers[name.toLowerCase()] = value+ },+ status(code: number) {+ this.statusCode = code+ return this+ },+ json(body: unknown) {+ resolve({ next: false, status: this.statusCode, body, headers })+ },+ }+ const req = { headers: { 'x-key': key } }+ middleware(req as never, res as never, (err?: unknown) =>+ err ? reject(err) : resolve({ next: true, headers })+ )+ })++ for (let n = 0; n < 100_000; n += 1) {+ const outcome = await call(`key:${n}`)+ if (!outcome.next) throw new Error(`key:${n} was not admitted: ${JSON.stringify(outcome)}`)+ }+ const overCap = await call('key:100000')+ const tracked = await call('key:0')++ expect(overCap).toEqual({+ next: false,+ status: 503,+ body: { error: 'rate_limit_unavailable', retryAfterSeconds: 2 },+ headers: { 'retry-after': '2', 'cache-control': 'no-store' },+ })+ // Witness: a key already tracked is still counted in memory (its second hit).+ expect(tracked).toEqual({+ next: true,+ headers: expect.objectContaining({ 'x-ratelimit-remaining': String(LIMIT - 2) }),+ })+ expect(checkAndIncrement).toHaveBeenCalledTimes(100_002)+ expect(hits.inc.mock.calls.at(-2)).toEqual([+ { bucket_type: 'unit_bucket', result: 'unavailable' },+ 1,+ ])+ expect(hits.inc.mock.calls.at(-1)).toEqual([+ { bucket_type: 'unit_bucket', result: 'fallback_allowed' },+ 1,+ ])+ }, 60_000)+it("'closed' still admits a counted request and still answers 429 over the limit", async () => {checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))const { app, handler } = appWith('closed')diff --git a/control-api/test/recipeOauth.userToken.test.ts b/control-api/test/recipeOauth.userToken.test.tsindex b979c9bc9..b143f719d 100644--- a/control-api/test/recipeOauth.userToken.test.ts+++ b/control-api/test/recipeOauth.userToken.test.ts@@ -26,6 +26,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -196,6 +197,7 @@ describe('routes/recipe-oauth — POST /recipe-oauth/user-token (SEC-5)', () =>seedRecipe(gateway, { name: 'leadforge', backgroundAccess: true })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30_000,windowStartMs: Date.now(),diff --git a/control-api/test/recipeOauth.users.test.ts b/control-api/test/recipeOauth.users.test.tsindex e7cae7169..096a4410a 100644--- a/control-api/test/recipeOauth.users.test.ts+++ b/control-api/test/recipeOauth.users.test.ts@@ -24,6 +24,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -138,8 +139,7 @@ describe('routes/recipe-oauth — GET /recipe-oauth/users (SEC-6)', () => {})it('401 without a broker token', async () => {- const res = await request(app)- .get('/api/v1/recipe-oauth/users?oauthClientId=google-gmail')+ const res = await request(app).get('/api/v1/recipe-oauth/users?oauthClientId=google-gmail')expect(res.status).toBe(401)})@@ -169,6 +169,7 @@ describe('routes/recipe-oauth — GET /recipe-oauth/users (SEC-6)', () => {seedRecipe(gateway, { name: 'leadforge', backgroundAccess: true })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts b/control-api/test/routes.adminAuth.publicTokenLimiter.test.tsnew file mode 100644index 000000000..f22ec20f8--- /dev/null+++ b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts@@ -0,0 +1,222 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import { config } from '../src/config.js'++/**+ * The public control-admin token routes (password reset, invitation, email+ * confirmation) are called before login. Each route has two buckets:+ *+ * - IP + submitted value (email, login or token prefix) at+ * `adminPublicTokenRlPerMin`: one person retrying the same value.+ * - IP alone at `adminPublicTokenIpRlPerMin`: the ceiling on rotating values+ * from one source. Many legitimate admins can share one public IP (a+ * corporate VPN egress, an office NAT), so this ceiling is several times the+ * per-value one, and a request the per-value bucket refuses is never charged+ * to the IP bucket: one person's retry loop cannot use up the office's budget.+ *+ * The counter is an in-memory stand-in for the Postgres bucket; the real+ * rateLimitMiddleware runs on top of it.+ */++const counts = vi.hoisted(() => new Map<string, number>())+const limiterCalls = vi.hoisted(() => [] as Array<{ key: string; max: number }>)++vi.mock('../src/services/rateLimiterService.js', () => ({+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,+ checkAndIncrement: vi.fn(async (key: string, max: number) => {+ limiterCalls.push({ key, max })+ const count = (counts.get(key) ?? 0) + 1+ counts.set(key, count)+ return {+ allowed: count <= max,+ remaining: Math.max(0, max - count),+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count,+ backendAvailable: true,+ }+ }),+}))++const registration = vi.hoisted(() => ({+ registerAndSendControlAdminPasswordReset: vi.fn(),+ validateControlAdminEmailConfirmationToken: vi.fn(),+ validateControlAdminInvitationToken: vi.fn(),+ validateControlAdminPasswordResetToken: vi.fn(),+}))+vi.mock('../src/services/controlAdminInvitationRegistrationService.js', () => registration)+vi.mock('../src/services/directory/index.js', () => ({+ acceptInvitationById: vi.fn(),+ getPendingMemberInvitationForEmail: vi.fn(),+ provisionAdminDesktopWorkspace: vi.fn(),+ setInvitationPasswordForUser: vi.fn(),+}))+vi.mock('../src/services/adminAuthService.js', () => ({+ completeControlAdminEmailChangeRequest: vi.fn(),+ completeControlAdminInvitation: vi.fn(),+ completeControlAdminPasswordResetRequest: vi.fn(),+ createControlAdminPasswordResetRequest: vi.fn(),+ findAdminById: vi.fn(),+ findAdminByLogin: vi.fn(),+ getPendingControlAdminEmailChangeRequest: vi.fn(),+ getPendingControlAdminInvitation: vi.fn(),+ getPendingControlAdminPasswordResetRequest: vi.fn(),+ isValidAdminEmail: vi.fn(() => true),+ isValidAdminUsername: vi.fn(() => true),+ markControlAdminInvitationOpened: vi.fn(),+ registerAdminFailedLogin: vi.fn(),+ registerAdminSuccessfulLogin: vi.fn(),+ revokeAdminTokenJti: vi.fn(),+ revokeControlAdminPasswordResetRequest: vi.fn(),+ setupInitialAdminCredentials: vi.fn(),+}))+vi.mock('../src/services/initialAdminSetupService.js', () => ({+ setupInitialAdminWithDesktopWorkspace: vi.fn(),+}))+vi.mock('../src/utils/auth/adminAuthToken.js', () => ({ signAdminToken: vi.fn() }))+vi.mock('../src/middleware/controlUIAuth.js', () => ({+ requireAuthForControlUI: (_req: unknown, _res: unknown, next: () => void) => next(),+}))++const { createAdminAuthRouter } = await import('../src/routes/admin/auth.js')++const PER_VALUE = 3+const PER_IP = 10+const VALIDATE = '/admin/auth/password-reset/validate'++const PUBLIC_TOKEN_ROUTES = [+ ['password_reset_request', '/admin/auth/password-reset/request'],+ ['password_reset_validate', '/admin/auth/password-reset/validate'],+ ['password_reset_complete', '/admin/auth/password-reset/complete'],+ ['invitation_validate', '/admin/auth/control-admin-invitations/validate'],+ ['invitation_complete', '/admin/auth/control-admin-invitations/complete'],+ ['email_confirmation_validate', '/admin/auth/control-admin-email-confirmations/validate'],+ ['email_confirmation_complete', '/admin/auth/control-admin-email-confirmations/complete'],+] as const++function buildApp() {+ const warn = vi.fn()+ const app = express()+ app.set('trust proxy', 1)+ app.use(express.json())+ app.use((req, _res, next) => {+ ;(req as unknown as { log: { warn: typeof warn } }).log = { warn }+ next()+ })+ app.use(createAdminAuthRouter())+ return { app, warn }+}++function validateFrom(app: express.Express, ip: string, email: string) {+ return request(app)+ .post(VALIDATE)+ .set('x-forwarded-for', ip)+ .send({ token: `token-${email}`, email })+}++function deniedBucketTypes(warn: ReturnType<typeof vi.fn>): string[] {+ return warn.mock.calls+ .map(([fields]) => fields as { event?: string; bucketType?: string })+ .filter(fields => fields.event === 'rate_limit_denied')+ .map(fields => fields.bucketType ?? '')+}++describe('public control-admin token limiter', () => {+ beforeEach(() => {+ counts.clear()+ limiterCalls.length = 0+ Object.values(registration).forEach(fn => fn.mockReset())+ // The handler's own outcome is irrelevant here; reaching it is the witness.+ registration.validateControlAdminPasswordResetToken.mockRejectedValue(new Error('invalid'))+ config.adminPublicTokenRlPerMin = PER_VALUE+ config.adminPublicTokenIpRlPerMin = PER_IP+ })++ it('lets every admin behind one shared public IP through, each with their own value', async () => {+ const { app, warn } = buildApp()+ const statuses: number[] = []+ for (let user = 0; user < PER_IP; user += 1) {+ statuses.push((await validateFrom(app, '203.0.113.7', `admin${user}@corp.test`)).status)+ }++ // More admins than the per-value limit, one request each, all from one IP.+ expect(statuses).toHaveLength(PER_IP)+ expect(statuses.every(status => status === 400)).toBe(true)+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP)+ expect(deniedBucketTypes(warn)).toEqual([])+ })++ it('refuses value rotation from one IP past the IP ceiling, and names the IP bucket', async () => {+ const { app, warn } = buildApp()+ for (let n = 0; n < PER_IP; n += 1) {+ expect((await validateFrom(app, '203.0.113.7', `guess${n}@corp.test`)).status).toBe(400)+ }++ const refused = await validateFrom(app, '203.0.113.7', `guess${PER_IP}@corp.test`)+ expect(refused.status).toBe(429)+ expect(refused.headers['x-ratelimit-limit']).toBe(String(PER_IP))+ expect(deniedBucketTypes(warn)).toEqual(['control_admin_public_ip_password_reset_validate'])+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP)++ // Another source is unaffected.+ expect((await validateFrom(app, '198.51.100.9', '[email protected]')).status).toBe(400)+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP + 1)+ })++ it('does not charge the IP bucket for a request the per-value bucket refused', async () => {+ const { app, warn } = buildApp()+ const retries: number[] = []+ for (let attempt = 0; attempt < PER_VALUE + 5; attempt += 1) {+ retries.push((await validateFrom(app, '203.0.113.7', '[email protected]')).status)+ }+ expect(retries.filter(status => status === 400)).toHaveLength(PER_VALUE)+ expect(retries.filter(status => status === 429)).toHaveLength(5)+ expect(new Set(deniedBucketTypes(warn))).toEqual(+ new Set(['control_admin_public_password_reset_validate'])+ )++ // The IP bucket saw only the PER_VALUE admitted requests, so the other+ // admins on that IP still have PER_IP - PER_VALUE requests.+ const others: number[] = []+ for (let user = 0; user < PER_IP - PER_VALUE; user += 1) {+ others.push((await validateFrom(app, '203.0.113.7', `colleague${user}@corp.test`)).status)+ }+ expect(others).toHaveLength(PER_IP - PER_VALUE)+ expect(others.every(status => status === 400)).toBe(true)+ expect((await validateFrom(app, '203.0.113.7', '[email protected]')).status).toBe(429)+ })++ it('groups IPv6 sources by /56, so rotating addresses inside one allocation does not evade the ceiling', async () => {+ const { app } = buildApp()+ for (let n = 0; n < PER_IP; n += 1) {+ // Every address differs, and all of them sit in 2001:db8:0:0000::/56.+ const status = (+ await validateFrom(app, `2001:db8:0:${n.toString(16)}::${n + 1}`, `v6-${n}@corp.test`)+ ).status+ expect(status).toBe(400)+ }+ expect((await validateFrom(app, '2001:db8:0:ff::1', '[email protected]')).status).toBe(429)+ // The next /56 is another source.+ expect((await validateFrom(app, '2001:db8:0:100::1', '[email protected]')).status).toBe(400)+ })++ it.each(PUBLIC_TOKEN_ROUTES)(+ '%s charges the per-value bucket, then the per-IP bucket, at the configured limits',+ async (routeName, path) => {+ const { app } = buildApp()+ await request(app)+ .post(path)+ .set('x-forwarded-for', '203.0.113.7')+ .send({ token: 'tok', email: '[email protected]', password: 'x'.repeat(12) })++ expect(limiterCalls).toEqual([+ {+ key: `control-admin-public:${routeName}:203.0.113.7:[email protected]`,+ max: PER_VALUE,+ },+ { key: `control-admin-public-ip:${routeName}:203.0.113.7`, max: PER_IP },+ ])+ }+ )+})diff --git a/control-api/test/routes.adminPluginWorkloadSdk.test.ts b/control-api/test/routes.adminPluginWorkloadSdk.test.tsindex 6f66c9648..51f9f9aee 100644--- a/control-api/test/routes.adminPluginWorkloadSdk.test.ts+++ b/control-api/test/routes.adminPluginWorkloadSdk.test.ts@@ -118,6 +118,7 @@ beforeEach(() => {vi.mocked(checkAndIncrement).mockReset()vi.mocked(checkAndIncrement).mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 119,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.adminRegistryGrants.test.ts b/control-api/test/routes.adminRegistryGrants.test.tsindex 960405ead..71975e810 100644--- a/control-api/test/routes.adminRegistryGrants.test.ts+++ b/control-api/test/routes.adminRegistryGrants.test.ts@@ -80,6 +80,7 @@ vi.mock('../src/services/registryConnectionDb.js', async importOriginal => {vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn(async () => ({allowed: true,+ backendAvailable: true,remaining: 29,resetMs: Date.now() + 60000,})),@@ -113,6 +114,7 @@ beforeEach(() => {connDb.isRegistryAuthActive.mockImplementation(async () => cfg.registryAuthEnabled)vi.mocked(checkAndIncrement).mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 29,resetMs: Date.now() + 60000,} as never)diff --git a/control-api/test/routes.adminRegistryKeys.test.ts b/control-api/test/routes.adminRegistryKeys.test.tsindex c56e1e9b3..6234a81cd 100644--- a/control-api/test/routes.adminRegistryKeys.test.ts+++ b/control-api/test/routes.adminRegistryKeys.test.ts@@ -57,6 +57,7 @@ vi.mock('../src/services/registryConnectionDb.js', () => connDb)vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn(async () => ({allowed: true,+ backendAvailable: true,remaining: 29,resetMs: Date.now() + 60000,})),@@ -90,6 +91,7 @@ beforeEach(() => {connDb.isRegistryAuthActive.mockImplementation(async () => cfg.registryAuthEnabled)vi.mocked(checkAndIncrement).mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 29,resetMs: Date.now() + 60000,} as never)@@ -188,6 +190,7 @@ describe('GET /admin/registry/keys', () => {it('429 when the per-admin rate limit is exceeded', async () => {vi.mocked(checkAndIncrement).mockResolvedValue({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 60000,} as never)diff --git a/control-api/test/routes.auth.codexSubscriptionCallback.test.ts b/control-api/test/routes.auth.codexSubscriptionCallback.test.tsindex 52ed15c3e..11811b4d4 100644--- a/control-api/test/routes.auth.codexSubscriptionCallback.test.ts+++ b/control-api/test/routes.auth.codexSubscriptionCallback.test.ts@@ -24,6 +24,7 @@ vi.mock('../src/db.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 19,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.externalGfs.grantsLimiterWiring.test.ts b/control-api/test/routes.externalGfs.grantsLimiterWiring.test.tsnew file mode 100644index 000000000..b233b0033--- /dev/null+++ b/control-api/test/routes.externalGfs.grantsLimiterWiring.test.ts@@ -0,0 +1,109 @@+import { describe, expect, it, vi } from 'vitest'+import type { Router } from 'express'++/**+ * #764: the two actor buckets in front of the external grant and share routes+ * refuse a request they cannot count. An HTTP test cannot observe this option,+ * because the pre-resolution Postgres bucket fails closed first on the same+ * pool, so the wiring is read from the router: which options each limiter was+ * built with, and which routes mount it.+ */++type LimiterOptions = { bucketType: string; onBackendUnavailable: 'process-memory' | 'closed' }+type TaggedLimiter = ((...a: unknown[]) => void) & { limiterOptions: LimiterOptions }++const built = vi.hoisted(() => [] as LimiterOptions[])++vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({+ rateLimitMiddleware: (options: LimiterOptions) => {+ built.push(options)+ const limiter = ((_req: unknown, _res: unknown, next: () => void) => next()) as TaggedLimiter+ limiter.limiterOptions = options+ return limiter+ },+}))+vi.mock('../src/config.js', () => ({+ config: {+ gfscBaseUrl: 'http://gfsc.gfs.svc:8087',+ gfscWriteBaseUrl: 'http://gfsc-writer.gfs.svc:8087',+ gfscProxyTimeoutMs: 300_000,+ gfsUploadMaxPartBytes: 16 * 1024 * 1024,+ hostsNamespace: 'mcp-host',+ desktopGfsOperatorLinkingEnabled: false,+ externalGfsIngressRlPerMin: 1800,+ externalGfsTokenUserRlPerMin: 10,+ externalGfsTokenIpRlPerMin: 600,+ externalGfsIpRlPerMin: 1200,+ externalGfsResourceReadRlPerMin: 120,+ externalGfsProxyReadRlPerMin: 60,+ externalGfsGrantsReadRlPerMin: 45,+ externalGfsSharesReadRlPerMin: 35,+ externalGfsOperationRlPerMin: 30,+ },+}))+// Both derive signing keys at module load; building the router never signs.+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ verifyExternalSessionToken: vi.fn(),+}))+vi.mock('../src/auth/gfsToken.js', () => ({+ GFS_DELETE_SCOPE: 'gfs.delete',+ GFS_READ_SCOPE: 'gfs.read',+ GFS_WRITE_SCOPE: 'gfs.write',+ GFS_SCOPES: ['gfs.read', 'gfs.write', 'gfs.delete', 'gfs.manage_acl', 'gfs.share'],+ signGfsToken: vi.fn(),+}))+vi.mock('../src/db.js', () => ({+ pool: { query: vi.fn() },+ rateLimitPool: { query: vi.fn() },+ withTransaction: vi.fn(),+}))+// grants.ts/shares.ts/token.ts import requireAuthForControlUI, which derives the+// admin JWT key at module load; these routes never call it.+vi.mock('../src/middleware/controlUIAuth.js', () => ({+ requireAuthForControlUI: (_req: unknown, _res: unknown, next: () => void) => next(),+}))++const { createExternalGfsRouter } = await import('../src/routes/external/gfs.js')++type RouteLayer = {+ route?: { path: string; methods: Record<string, boolean>; stack: Array<{ handle: unknown }> }+}++function actorLimiterOf(router: Router, method: string, path: string): LimiterOptions {+ const layer = (router.stack as RouteLayer[]).find(+ l => l.route?.path === path && l.route.methods[method] === true+ )+ if (!layer?.route) throw new Error(`route ${method.toUpperCase()} ${path} is not mounted`)+ const limiters = layer.route.stack+ .map(s => (s.handle as Partial<TaggedLimiter>).limiterOptions)+ .filter((o): o is LimiterOptions => o !== undefined)+ expect(limiters).toHaveLength(1)+ return limiters[0]!+}++describe('external GFS grant and share actor limiters', () => {+ const router = createExternalGfsRouter()++ it('builds both actor buckets to fail closed', () => {+ // Witness: the router built exactly these two limiters, so the rows below+ // are the options the routes use and not an empty match.+ expect(built.map(o => o.bucketType).sort()).toEqual([+ 'gfs_grants_external',+ 'gfs_grants_external_read',+ ])+ for (const options of built) expect(options.onBackendUnavailable).toBe('closed')+ })++ it.each([+ ['get', '/external/gfs/grants', 'gfs_grants_external_read'],+ ['get', '/external/gfs/shares', 'gfs_grants_external_read'],+ ['put', '/external/gfs/grants', 'gfs_grants_external'],+ ['delete', '/external/gfs/grants/:id', 'gfs_grants_external'],+ ['post', '/external/gfs/shares', 'gfs_grants_external'],+ ['delete', '/external/gfs/shares/:id', 'gfs_grants_external'],+ ])('%s %s mounts %s, which fails closed', (method, path, bucketType) => {+ const limiter = actorLimiterOf(router, method, path)+ expect(limiter.bucketType).toBe(bucketType)+ expect(limiter.onBackendUnavailable).toBe('closed')+ })+})diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.tsindex 8c5bf1b53..8e11bfa78 100644--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts@@ -261,7 +261,7 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =await corePool.query('DROP FUNCTION IF EXISTS t3b_limiter_fault()')}- it('T3b: the external grants limiter fails closed on a backend error while an open-policy route still passes', async () => {+ it('T3b: the external grants limiter fails closed on a backend error while a process-memory route still passes', async () => {const app = mod.createApp(new mod.MockGateway())const userId = await seedUser()const internalToken = mod.config.internalServiceTokens['external-rest-api']@@ -292,8 +292,9 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =expect(grants.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })expect(grants.headers['retry-after']).toBe('2')expect(grants.headers['cache-control']).toBe('no-store')- // Witness for the open policy under the same fault: the ack route's- // limiter could not count either, and its handler still answered.+ // Witness for the process-memory policy under the same fault: the ack+ // route's Postgres limiter could not count either, its in-memory counter+ // admitted the first request, and its handler answered.expect(ack.status).toBe(404)expect(ack.body).toEqual({ error: 'notification_not_found' })diff --git a/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts b/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.tsindex b1b430f3c..245de369a 100644--- a/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts+++ b/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts@@ -27,6 +27,7 @@ vi.mock('../src/db.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -281,6 +282,7 @@ describe('POST /api/v1/internal/mcp-oauth/authorize-url (U5)', () => {seedOauthServer(gateway, { name: 'gdrive', grantScope: 'user' })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts b/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.tsindex 3522bb192..3a77b9a12 100644--- a/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts+++ b/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts@@ -28,6 +28,7 @@ vi.mock('../src/db.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.internal.mcpOauthDisconnect.test.ts b/control-api/test/routes.internal.mcpOauthDisconnect.test.tsindex e9b43a1b9..44552a2d2 100644--- a/control-api/test/routes.internal.mcpOauthDisconnect.test.ts+++ b/control-api/test/routes.internal.mcpOauthDisconnect.test.ts@@ -45,6 +45,7 @@ vi.mock('../src/db.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -258,6 +259,7 @@ describe('DELETE /api/v1/internal/mcp-oauth/grant (spec 11 U4)', () => {seedOauthServer(gateway, { name: 'gdrive' })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.tsindex 99653a8a3..4100f5574 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts@@ -10,6 +10,7 @@ import * as mcpHostJwt from '../src/utils/auth/mcpHostJwtToken.js'vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.tsindex 667296841..36a0b11b7 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts@@ -28,6 +28,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.test.tsindex 066407bbd..9760e36a2 100644--- a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts@@ -13,6 +13,7 @@ import * as mcpHostJwt from '../src/utils/auth/mcpHostJwtToken.js'vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.mcp-host.workflows.test.ts b/control-api/test/routes.mcp-host.workflows.test.tsindex 52fd88e43..5e9da9e38 100644--- a/control-api/test/routes.mcp-host.workflows.test.ts+++ b/control-api/test/routes.mcp-host.workflows.test.ts@@ -61,6 +61,7 @@ vi.mock('../src/services/adminAuthService.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 10,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.mcpOauth.test.ts b/control-api/test/routes.mcpOauth.test.tsindex c6c4ba9f6..f84fcaffb 100644--- a/control-api/test/routes.mcpOauth.test.ts+++ b/control-api/test/routes.mcpOauth.test.ts@@ -24,6 +24,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -608,6 +609,7 @@ describe('routes/mcp-oauth — POST /mcp-oauth/grants/exists (mini-spec 13)', ()seedOauthServer(gateway, { name: 'gdrive', grantScope: 'user' })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.profile.test.ts b/control-api/test/routes.profile.test.tsindex 042ea4596..ee5a3ac86 100644--- a/control-api/test/routes.profile.test.ts+++ b/control-api/test/routes.profile.test.ts@@ -132,6 +132,7 @@ describe('routes/profile', () => {rateLimitMock.checkAndIncrement.mockReset()rateLimitMock.checkAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.ts b/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.tsnew file mode 100644index 000000000..99ea343a0--- /dev/null+++ b/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.ts@@ -0,0 +1,231 @@+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool, type PoolClient } from 'pg'+import request from 'supertest'++// rateLimitMiddleware's 'process-memory' mode against real PostgreSQL: with+// every limiter pool connection held, a real route's limiter cannot count in+// Postgres and counts in process memory instead. Requests go through the+// production Express app; the ONLY mock is the session-token verifier+// (identity), as in routes.externalGfsRateLimitFailClosed. Skipped without+// CONTROL_API_REAL_PG_ADMIN_URL.+//+// The limiter pool is small with a short acquire timeout, and the external+// per-operation limit is low, so each held-pool request waits 200 ms instead+// of the production 5 s and the suite needs four of them, not sixty-one.++vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ // A token that is not a JSON identity fails verification, as a bad+ // signature does in production.+ verifyExternalSessionToken: (token: string) => {+ if (!token.startsWith('{')) return null+ const parsed = JSON.parse(token) as { userId: string }+ return {+ userId: parsed.userId,+ email: `${parsed.userId}@example.test`,+ teamId: null,+ role: 'member',+ authGeneration: 1,+ exp: Math.floor(Date.now() / 1000) + 3600,+ }+ },+}))++const LIMITER_POOL_MAX = 2+const LIMITER_ACQUIRE_TIMEOUT_MS = 200+const EXTERNAL_PER_MIN = 3++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++describeRealPostgres('rateLimitMiddleware process-memory mode (real PostgreSQL)', () => {+ const database = `control_api_rl_process_memory_${randomBytes(6).toString('hex')}`+ const envKeys = [+ 'CONTROL_API_PG_CONNECTION_STRING',+ 'CORE_POOL_MAX',+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_MAX',+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',+ 'APPROVAL_RL_EXTERNAL_PER_MIN',+ ] as const+ const previousEnv = new Map<string, string | undefined>()++ let adminPool: Pool+ let corePool: Pool+ let limiterPool: Pool+ let mod: {+ createApp: (gateway: unknown) => import('express').Express+ config: typeof import('../src/config.js').config+ hitsTotal: typeof import('../src/observability/metrics.js').rateLimitHitsTotal+ MockGateway: typeof import('./mockGateway.js').MockGateway+ }++ beforeAll(async () => {+ if (adminUrl === undefined) {+ throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')+ }+ const connectionString = databaseUrl(adminUrl, database)+ for (const key of envKeys) {+ previousEnv.set(key, process.env[key])+ delete process.env[key]+ }+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString+ process.env.RATE_LIMIT_POOL_MAX = String(LIMITER_POOL_MAX)+ process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = String(LIMITER_ACQUIRE_TIMEOUT_MS)+ process.env.APPROVAL_RL_EXTERNAL_PER_MIN = String(EXTERNAL_PER_MIN)++ const dbMod = await import('../src/db.js')+ corePool = dbMod.pool as unknown as Pool+ corePool.on('error', () => {})+ limiterPool = dbMod.rateLimitPool as unknown as Pool+ limiterPool.on('error', () => {})+ const migratePool = new Pool({ connectionString })+ await dbMod.initDb({ connect: () => migratePool.connect() })+ await migratePool.end()++ const appMod = await import('../src/app.js')+ const configMod = await import('../src/config.js')+ const metricsMod = await import('../src/observability/metrics.js')+ const { MockGateway } = await import('./mockGateway.js')+ mod = {+ createApp: appMod.createApp as never,+ config: configMod.config,+ hitsTotal: metricsMod.rateLimitHitsTotal,+ MockGateway,+ }+ }, 60_000)++ afterAll(async () => {+ for (const key of envKeys) {+ const value = previousEnv.get(key)+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ await corePool?.end().catch(() => {})+ await limiterPool?.end().catch(() => {})+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ async function seedUser(): Promise<string> {+ const id = randomUUID()+ await corePool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [+ id,+ `${id}@example.test`,+ 'process-memory',+ ])+ return id+ }++ async function ack(app: import('express').Express, userId: string) {+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ return request(app)+ .post(`/api/v1/external/notifications/${randomUUID()}/ack`)+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', JSON.stringify({ userId }))+ .set('x-forwarded-for', '203.0.113.7')+ }++ async function externalUserHits(): Promise<Record<string, number>> {+ const { values } = await mod.hitsTotal.get()+ const totals: Record<string, number> = {}+ for (const sample of values) {+ if (sample.labels.bucket_type !== 'external_user') continue+ const result = String(sample.labels.result)+ totals[result] = (totals[result] ?? 0) + sample.value+ }+ return totals+ }++ function delta(after: Record<string, number>, before: Record<string, number>, result: string) {+ return (after[result] ?? 0) - (before[result] ?? 0)+ }++ /** Read through the core pool: the limiter pool is the one being held. */+ async function ledgerCount(bucketKey: string): Promise<number | null> {+ const result = await corePool.query<{ count: number }>(+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1',+ [bucketKey]+ )+ return result.rows[0]?.count ?? null+ }++ it('T5: with every limiter connection held, the ack route admits max requests from process memory and answers 429 to the next; after release Postgres decides again', async () => {+ expect(mod.config.approvalRlExternalPerMin).toBe(EXTERNAL_PER_MIN)+ const app = mod.createApp(new mod.MockGateway())+ const userId = await seedUser()+ const bucketKey = `user:${userId}:notification-ack`+ await corePool.query('DELETE FROM rate_limit_buckets')+ const hitsBefore = await externalUserHits()++ const held: PoolClient[] = []+ const responses: request.Response[] = []+ try {+ for (let index = 0; index < LIMITER_POOL_MAX; index += 1) {+ held.push(await limiterPool.connect())+ }+ // Witness: the pool is exhausted, not merely busy.+ expect(limiterPool.totalCount).toBe(LIMITER_POOL_MAX)+ expect(limiterPool.idleCount).toBe(0)++ for (let n = 0; n <= EXTERNAL_PER_MIN; n += 1) responses.push(await ack(app, userId))+ } finally {+ for (const client of held) client.release()+ }++ // The handler ran for the first max requests: an unknown id is a 404.+ for (const admitted of responses.slice(0, EXTERNAL_PER_MIN)) {+ expect(admitted.status).toBe(404)+ expect(admitted.body).toEqual({ error: 'notification_not_found' })+ }+ const denied = responses[EXTERNAL_PER_MIN]!+ expect(denied.status).toBe(429)+ const retryAfterSeconds = Number(denied.headers['retry-after'])+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)+ expect(retryAfterSeconds).toBeLessThanOrEqual(60)+ expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })+ expect(denied.headers['x-ratelimit-remaining']).toBe('0')++ const hitsHeld = await externalUserHits()+ expect(delta(hitsHeld, hitsBefore, 'fallback_allowed')).toBe(EXTERNAL_PER_MIN)+ expect(delta(hitsHeld, hitsBefore, 'fallback_denied')).toBe(1)+ expect(delta(hitsHeld, hitsBefore, 'allowed')).toBe(0)+ expect(await ledgerCount(bucketKey)).toBeNull()++ // The in-memory counter for this key is over the limit, so an admitted+ // request now proves Postgres decided it.+ const recovered = await ack(app, userId)+ expect(recovered.status).toBe(404)+ expect(recovered.body).toEqual({ error: 'notification_not_found' })+ expect(recovered.headers['x-ratelimit-remaining']).toBe(String(EXTERNAL_PER_MIN - 1))+ // Liveness witness for the empty ledger above: the same request, once the+ // pool is free, is counted in rate_limit_buckets.+ expect(await ledgerCount(bucketKey)).toBe(1)+ const hitsAfter = await externalUserHits()+ expect(delta(hitsAfter, hitsHeld, 'allowed')).toBe(1)+ expect(delta(hitsAfter, hitsHeld, 'fallback_allowed')).toBe(0)+ expect(delta(hitsAfter, hitsHeld, 'fallback_denied')).toBe(0)+ }, 30_000)+})diff --git a/control-api/test/routes.recipeOauth.test.ts b/control-api/test/routes.recipeOauth.test.tsindex 8c45d093f..06fcbdc44 100644--- a/control-api/test/routes.recipeOauth.test.ts+++ b/control-api/test/routes.recipeOauth.test.ts@@ -26,6 +26,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({vi.mock('../src/services/rateLimiterService.js', () => ({checkAndIncrement: vi.fn().mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -202,6 +203,7 @@ describe('routes/recipe-oauth — POST /recipe-oauth/token', () => {seedRecipe(gateway, { name: 'crm', backgroundAccess: true })vi.mocked(checkAndIncrement).mockResolvedValueOnce({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30_000,windowStartMs: Date.now(),diff --git a/control-api/test/routes.registryConnect.test.ts b/control-api/test/routes.registryConnect.test.tsindex 4d1e870f1..07b681dad 100644--- a/control-api/test/routes.registryConnect.test.ts+++ b/control-api/test/routes.registryConnect.test.ts@@ -117,6 +117,7 @@ beforeEach(() => {cfg.registryUrl = 'https://example.com'rateLimiter.checkAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60000,})@@ -521,6 +522,7 @@ describe('registry connect flow', () => {it('GET 429s when the per-admin bucket is exceeded, without touching the registry', async () => {rateLimiter.checkAndIncrement.mockResolvedValue({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30000,})@@ -558,6 +560,7 @@ describe('registry connect flow', () => {connDb.getRegistryConnection.mockResolvedValue(null)rateLimiter.checkAndIncrement.mockResolvedValue({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30000,})@@ -1031,6 +1034,7 @@ describe('POST recover', () => {})rateLimiter.checkAndIncrement.mockResolvedValue({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30000,})diff --git a/control-api/test/routes.rpcAccessHostsWake.test.ts b/control-api/test/routes.rpcAccessHostsWake.test.tsindex 0b610cf9c..94288cac0 100644--- a/control-api/test/routes.rpcAccessHostsWake.test.ts+++ b/control-api/test/routes.rpcAccessHostsWake.test.ts@@ -81,6 +81,7 @@ describe('POST /rpc/hosts/:hostRef/wake', () => {authMock.verifyRpcAccessToken.mockReturnValue(makeClaims())rateLimiterMock.checkAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: 0,@@ -128,6 +129,7 @@ describe('POST /rpc/hosts/:hostRef/wake', () => {it('returns 429 with Retry-After when over the per-host limit, with no mutation', async () => {rateLimiterMock.checkAndIncrement.mockResolvedValue({allowed: false,+ backendAvailable: true,remaining: 0,resetMs: Date.now() + 30_000,windowStartMs: 0,diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.tsindex 4d3003f6b..e05695b0f 100644--- a/control-api/test/workflows.shared.rateLimit.test.ts+++ b/control-api/test/workflows.shared.rateLimit.test.ts@@ -60,6 +60,7 @@ function signedClaims(sub: string) {function pgAllows() {mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -204,6 +205,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -433,6 +435,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -462,6 +465,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -492,6 +496,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -520,6 +525,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -550,6 +556,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 59,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -679,6 +686,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -718,6 +726,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -748,6 +757,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -779,6 +789,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),@@ -818,6 +829,7 @@ describe('routes/workflows/shared/rateLimit', () => {mockCheckAndIncrement.mockReset()mockCheckAndIncrement.mockResolvedValue({allowed: true,+ backendAvailable: true,remaining: 9,resetMs: Date.now() + 60_000,windowStartMs: Date.now(),