Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(control-api): count requests in process memory while the Postgres limiter is unavailable

Alfredo Lopeza9964ddc8ac7
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 9479475ab..deae46cb8 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -308,6 +308,7 @@ jobs:
services.llmCatalogSync.realPostgres.integration \
routes.externalGfsRateLimitStress.realPostgres.integration \
routes.externalGfsRateLimitFailClosed.realPostgres.integration \
+ routes.rateLimitProcessMemory.realPostgres.integration \
services.rateLimiterPoolSaturation.realPostgres.integration
do
grep -Fq "$suite" "$normalized_log" || {
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 8090d616a..f1b260b97 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.617",
+ "version": "0.1.618",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.617",
+ "version": "0.1.618",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 9e658aedb..2bee1119c 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.617",
+ "version": "0.1.618",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
index 58f9d9ebd..57bd92721 100644
--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
@@ -32,7 +32,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {
// ENV-tunable platform limit (issue #348): CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN.
maxPerMinute: config.pluginSdkRequestBucketRlPerMin,
getBucketKey: pluginWorkloadSdkRequestBucketKey,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -46,7 +46,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {
if (!claims) return 'plugin_workload_sdk_internal:unauthenticated'
return `plugin_workload_sdk_internal:${claims.iss}:${claims.sub}`
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -64,6 +64,6 @@ export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
diff --git a/control-api/src/middleware/processMemoryRateLimiter.ts b/control-api/src/middleware/processMemoryRateLimiter.ts
new file mode 100644
index 000000000..86c69b992
--- /dev/null
+++ b/control-api/src/middleware/processMemoryRateLimiter.ts
@@ -0,0 +1,62 @@
+import { MemoryStore, type Options } from 'express-rate-limit'
+
+/** Distinct keys one counter accepts per window before it refuses new ones. */
+export const PROCESS_MEMORY_MAX_TRACKED_KEYS = 100_000
+
+const WINDOW_MS = 60_000
+
+export type ProcessMemoryDecision =
+ | { outcome: 'allowed' | 'denied'; totalHits: number; resetMs: number }
+ | { outcome: 'key_cap_reached' }
+
+/**
+ * A per-minute counter held in this process's memory. `rateLimitMiddleware`
+ * uses it only while the Postgres limiter cannot count a request
+ * (`onBackendUnavailable: 'process-memory'`).
+ *
+ * Each key gets a fixed 60 s window that starts at its first hit
+ * (express-rate-limit's `MemoryStore`). A request is allowed while the key's
+ * hit count in its window is at most `maxPerMinute`.
+ *
+ * `MemoryStore` has no key count, so this class records the distinct keys it
+ * has seen in its own 60 s window, and refuses a new key once that set holds
+ * `maxTrackedKeys`. The store keeps a key until its cleanup timer has run
+ * twice without a hit on it, so it can hold a few times `maxTrackedKeys`
+ * entries at once; the cap bounds how fast that grows.
+ */
+export class ProcessMemoryRateLimiter {
+ private readonly store = new MemoryStore()
+ private readonly keysThisWindow = new Set<string>()
+ private windowStartMs = Date.now()
+
+ constructor(
+ private readonly maxPerMinute: number,
+ private readonly maxTrackedKeys: number = PROCESS_MEMORY_MAX_TRACKED_KEYS
+ ) {
+ // MemoryStore.init reads only `windowMs` from the options. It starts an
+ // unref'd cleanup timer, so an idle counter does not keep the process up.
+ this.store.init({ windowMs: WINDOW_MS } as Options)
+ }
+
+ async hit(key: string): Promise<ProcessMemoryDecision> {
+ const now = Date.now()
+ if (now - this.windowStartMs >= WINDOW_MS) {
+ this.keysThisWindow.clear()
+ this.windowStartMs = now
+ }
+ if (!this.keysThisWindow.has(key)) {
+ if (this.keysThisWindow.size >= this.maxTrackedKeys) return { outcome: 'key_cap_reached' }
+ this.keysThisWindow.add(key)
+ }
+
+ const { totalHits, resetTime } = await this.store.increment(key)
+ if (resetTime === undefined) {
+ throw new Error('express-rate-limit MemoryStore returned a hit without a reset time')
+ }
+ return {
+ outcome: totalHits <= this.maxPerMinute ? 'allowed' : 'denied',
+ totalHits,
+ resetMs: resetTime.getTime(),
+ }
+ }
+}
diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.ts
index 63f01c186..0bcd39d94 100644
--- a/control-api/src/middleware/rateLimitMiddleware.ts
+++ b/control-api/src/middleware/rateLimitMiddleware.ts
@@ -1,10 +1,12 @@
import { NextFunction, Request, Response } from 'express'
import { createHash } from 'node:crypto'
+import { LogThrottle } from '../observability/logThrottle.js'
import { rateLimitHitsTotal } from '../observability/metrics.js'
import {
RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
checkAndIncrement,
} from '../services/rateLimiterService.js'
+import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'
/**
* Factory for a per-request rate limit middleware backed by a PG token bucket.
@@ -15,6 +17,7 @@ import {
* bucketType: "recipe",
* maxPerMinute: 10,
* getBucketKey: (req) => mcpHostRateLimitBucketKey('recipe', req.mcpHostJwt),
+ * onBackendUnavailable: 'process-memory',
* }),
* handler,
* );
@@ -22,21 +25,90 @@ import {
* The middleware:
* - Returns 429 with `Retry-After` (seconds until window end) when the limit
* is exceeded.
- * - Emits `rate_limit_hits_total{bucket_type, result}` on every call.
+ * - Emits `rate_limit_hits_total{bucket_type, result}` on every call, with
+ * `result` one of `allowed`, `denied`, `unavailable`, `fallback_allowed`,
+ * `fallback_denied`.
* - If `getBucketKey` returns `null` (e.g. unable to derive key), the
* request is *allowed* but not counted — fail-open. Callers that need
* strict enforcement should return a sentinel key instead.
* - When the limiter backend cannot count the request
- * (`backendAvailable: false`), `onBackendUnavailable` decides: `'open'`
- * lets it through uncounted, `'closed'` answers 503 with `Retry-After`.
- * The option is required so every caller states that choice.
+ * (`backendAvailable: false`: the limiter pool is saturated, Postgres
+ * returned an error, or the upsert returned no row), `onBackendUnavailable`
+ * decides. The option is required so every caller states that choice.
+ * - `'closed'` answers 503 `rate_limit_unavailable` with `Retry-After`.
+ * - `'process-memory'` counts the request in this middleware instance's
+ * own in-memory counter: the same `maxPerMinute`, in a 60 s window per
+ * key. Over the limit it answers 429 like a Postgres denial, and the
+ * warn line carries `source: 'process-memory'`. The in-memory counter
+ * is used only for requests Postgres could not count; the two counts
+ * are never added together. It tracks at most 100 000 keys per 60 s;
+ * a new key beyond that answers 503 as `'closed'` does. The counter
+ * lives in one process: control-api runs one replica today, so the
+ * in-memory limit holds for the whole service only while that is
+ * true. With N replicas a caller can get N × `maxPerMinute` requests
+ * per minute while Postgres is unavailable.
*/
export function rateLimitMiddleware(opts: {
bucketType: string
maxPerMinute: number
getBucketKey: (req: Request) => string | null
- onBackendUnavailable: 'open' | 'closed'
+ onBackendUnavailable: 'process-memory' | 'closed'
}) {
+ // While the backend is down every request on this bucket fails the same way,
+ // so the warn line is written once per minute; the counter records each one.
+ const unavailableLogThrottle = new LogThrottle(60_000)
+ const processMemory =
+ opts.onBackendUnavailable === 'process-memory'
+ ? new ProcessMemoryRateLimiter(opts.maxPerMinute)
+ : null
+
+ function answerUnavailable(req: Request, res: Response): void {
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)
+ const suppressed = unavailableLogThrottle.admit(opts.bucketType)
+ if (req.log && suppressed !== undefined) {
+ req.log.warn(
+ { event: 'rate_limit_unavailable', bucketType: opts.bucketType, suppressed },
+ 'rate limit backend unavailable'
+ )
+ }
+ res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
+ res.setHeader('Cache-Control', 'no-store')
+ res.status(503).json({
+ error: 'rate_limit_unavailable',
+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ })
+ }
+
+ function answerDenied(
+ req: Request,
+ res: Response,
+ key: string,
+ denial: { count: number; resetMs: number; source?: 'process-memory' }
+ ): void {
+ const retryAfterSec = Math.max(1, Math.ceil((denial.resetMs - Date.now()) / 1000))
+ res.setHeader('Retry-After', String(retryAfterSec))
+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
+ res.setHeader('X-RateLimit-Remaining', '0')
+ res.setHeader('X-RateLimit-Reset', String(Math.floor(denial.resetMs / 1000)))
+ if (req.log) {
+ req.log.warn(
+ {
+ event: 'rate_limit_denied',
+ bucketType: opts.bucketType,
+ // Keys carry user ids (e.g. desktopUserId). The SHA-256 is a
+ // correlation id, not anonymization: it is unsalted, so a
+ // low-entropy key (an IP, an email) can be recovered by guessing.
+ hashedKey: createHash('sha256').update(key).digest('hex'),
+ count: denial.count,
+ maxPerMinute: opts.maxPerMinute,
+ ...(denial.source === undefined ? {} : { source: denial.source }),
+ },
+ 'rate limit exceeded'
+ )
+ }
+ res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })
+ }
+
return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {
void (async () => {
try {
@@ -49,43 +121,35 @@ export function rateLimitMiddleware(opts: {
}
const result = await checkAndIncrement(key, opts.maxPerMinute)
- if (!result.backendAvailable && opts.onBackendUnavailable === 'closed') {
- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'unavailable' }, 1)
- if (req.log) {
- req.log.warn(
- { event: 'rate_limit_unavailable', bucketType: opts.bucketType },
- 'rate limit backend unavailable'
- )
+ if (!result.backendAvailable) {
+ if (processMemory === null) {
+ answerUnavailable(req, res)
+ return
+ }
+ const decision = await processMemory.hit(key)
+ if (decision.outcome === 'key_cap_reached') {
+ answerUnavailable(req, res)
+ return
}
- res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
- res.setHeader('Cache-Control', 'no-store')
- res.status(503).json({
- error: 'rate_limit_unavailable',
- retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
- })
+ if (decision.outcome === 'denied') {
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)
+ answerDenied(req, res, key, {
+ count: decision.totalHits,
+ resetMs: decision.resetMs,
+ source: 'process-memory',
+ })
+ return
+ }
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)
+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
+ res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))
+ res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))
+ next()
return
}
if (!result.allowed) {
rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)
- const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))
- res.setHeader('Retry-After', String(retryAfterSec))
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', '0')
- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
- if (req.log) {
- req.log.warn(
- {
- event: 'rate_limit_denied',
- bucketType: opts.bucketType,
- // Keys carry user ids (e.g. desktopUserId); log their SHA-256.
- hashedKey: createHash('sha256').update(key).digest('hex'),
- count: result.count,
- maxPerMinute: opts.maxPerMinute,
- },
- 'rate limit exceeded'
- )
- }
- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })
+ answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })
return
}
diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.ts
index e8b976352..c40efc41b 100644
--- a/control-api/src/routes/admin/auth.ts
+++ b/control-api/src/routes/admin/auth.ts
@@ -1,5 +1,6 @@
-import { Router } from 'express'
+import { type Request, type RequestHandler, Router } from 'express'
import bcrypt from 'bcryptjs'
+import { ipKeyGenerator } from 'express-rate-limit'
import { config } from '../../config.js'
import { pool } from '../../db.js'
import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'
@@ -50,8 +51,26 @@ const ADMIN_PASSWORD_MAX_LENGTH = 256
const logger = rootLogger.child({ module: 'admin-auth' })
-function publicAdminTokenRateLimit(routeName: string) {
- return rateLimitMiddleware({
+function remoteAddressOf(req: Request): string {
+ return req.ip || req.socket.remoteAddress || 'unknown'
+}
+
+/**
+ * Two buckets per public token route, applied in this order:
+ *
+ * 1. Source IP + submitted value (email, login or token prefix), at
+ * `adminPublicTokenRlPerMin`: one person retrying the same value.
+ * 2. Source IP alone (IPv6 grouped by /56), at `adminPublicTokenIpRlPerMin`:
+ * the ceiling on rotating values from one source, which the first bucket
+ * cannot see because every new value gets a fresh key.
+ *
+ * Many admins can share one public IP (a corporate VPN egress, an office
+ * NAT), so the second ceiling is several times the first. A request the first
+ * bucket refuses never reaches the second, so one person's retry loop costs
+ * the shared IP at most `adminPublicTokenRlPerMin` per minute.
+ */
+function publicAdminTokenRateLimits(routeName: string): [RequestHandler, RequestHandler] {
+ const perValue = rateLimitMiddleware({
bucketType: `control_admin_public_${routeName}`,
maxPerMinute: config.adminPublicTokenRlPerMin,
getBucketKey: req => {
@@ -60,11 +79,18 @@ function publicAdminTokenRateLimit(routeName: string) {
.trim()
.toLowerCase()
const login = String(req.body?.username || req.body?.login || '').trim()
- const remote = req.ip || req.socket.remoteAddress || 'unknown'
- return `control-admin-public:${routeName}:${remote}:${email || login || token.slice(0, 48) || 'missing'}`
+ return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${email || login || token.slice(0, 48) || 'missing'}`
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
+ })
+ const perIp = rateLimitMiddleware({
+ bucketType: `control_admin_public_ip_${routeName}`,
+ maxPerMinute: config.adminPublicTokenIpRlPerMin,
+ getBucketKey: req =>
+ `control-admin-public-ip:${routeName}:${ipKeyGenerator(remoteAddressOf(req))}`,
+ onBackendUnavailable: 'process-memory',
})
+ return [perValue, perIp]
}
export function createAdminAuthRouter(): Router {
@@ -240,7 +266,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/password-reset/request',
- publicAdminTokenRateLimit('password_reset_request'),
+ ...publicAdminTokenRateLimits('password_reset_request'),
async (req, res, next) => {
try {
const username = String(req.body?.username || '').trim()
@@ -276,7 +302,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/password-reset/validate',
- publicAdminTokenRateLimit('password_reset_validate'),
+ ...publicAdminTokenRateLimits('password_reset_validate'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
@@ -313,7 +339,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/password-reset/complete',
- publicAdminTokenRateLimit('password_reset_complete'),
+ ...publicAdminTokenRateLimits('password_reset_complete'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
@@ -370,7 +396,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/control-admin-invitations/validate',
- publicAdminTokenRateLimit('invitation_validate'),
+ ...publicAdminTokenRateLimits('invitation_validate'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
@@ -409,7 +435,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/control-admin-invitations/complete',
- publicAdminTokenRateLimit('invitation_complete'),
+ ...publicAdminTokenRateLimits('invitation_complete'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
@@ -501,7 +527,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/control-admin-email-confirmations/validate',
- publicAdminTokenRateLimit('email_confirmation_validate'),
+ ...publicAdminTokenRateLimits('email_confirmation_validate'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
@@ -541,7 +567,7 @@ export function createAdminAuthRouter(): Router {
router.post(
'/admin/auth/control-admin-email-confirmations/complete',
- publicAdminTokenRateLimit('email_confirmation_complete'),
+ ...publicAdminTokenRateLimits('email_confirmation_complete'),
async (req, res, next) => {
try {
const token = String(req.body?.token || '').trim()
diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.ts
index eee584e8b..0b098f0fe 100644
--- a/control-api/src/routes/admin/registry.ts
+++ b/control-api/src/routes/admin/registry.ts
@@ -3166,7 +3166,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orgkeys:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
// Guards ⓪①② — returns null after sending an error response.
@@ -3288,7 +3288,7 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orggrants:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
// Resolve the caller's own org for a self-service op, or send the terminal error
diff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.ts
index 25357ad1f..35811c028 100644
--- a/control-api/src/routes/admin/registryConnect.ts
+++ b/control-api/src/routes/admin/registryConnect.ts
@@ -223,7 +223,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_status:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
@@ -334,7 +334,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_request:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
@@ -593,7 +593,7 @@ export function createRegistryConnectRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_recover:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.ts
index e50c500ad..e8dfb0aa8 100644
--- a/control-api/src/routes/external/auth.ts
+++ b/control-api/src/routes/external/auth.ts
@@ -129,7 +129,7 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {
.toLowerCase()
return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/decisions.routes.ts b/control-api/src/routes/external/decisions.routes.ts
index 4967cd6c8..d72e50533 100644
--- a/control-api/src/routes/external/decisions.routes.ts
+++ b/control-api/src/routes/external/decisions.routes.ts
@@ -63,7 +63,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req, res, next) => {
void (async () => {
@@ -103,7 +103,7 @@ export function createExternalUserApprovalDecisionsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
requireApprovalDecisionAccess(),
(req, res, next) => {
diff --git a/control-api/src/routes/external/invitations.ts b/control-api/src/routes/external/invitations.ts
index a7e8c0c75..bacba9eba 100644
--- a/control-api/src/routes/external/invitations.ts
+++ b/control-api/src/routes/external/invitations.ts
@@ -46,7 +46,7 @@ export function createExternalInvitationsRouter(): Router {
bucketType: 'external_invitation_lookup',
maxPerMinute: 30,
getBucketKey: req => invitationLookupIpKey(req),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
@@ -80,7 +80,7 @@ export function createExternalInvitationsRouter(): Router {
bucketType: 'external_invitation_password_token',
maxPerMinute: 10,
getBucketKey: req => invitationLookupIpKey(req),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/external/notifications.routes.ts b/control-api/src/routes/external/notifications.routes.ts
index 23f25adfe..0ab0bba9e 100644
--- a/control-api/src/routes/external/notifications.routes.ts
+++ b/control-api/src/routes/external/notifications.routes.ts
@@ -111,7 +111,7 @@ export function createExternalNotificationsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}:notifications` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req: Request, res: Response) => {
void (async () => {
@@ -345,7 +345,7 @@ export function createExternalNotificationsRouter(): Router {
const uid = extReq.externalAuth?.userId
return uid ? `user:${uid}:notification-ack` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
asyncHandler(async (req: Request, res: Response) => {
const extReq = req as ExternalAuthedRequest
diff --git a/control-api/src/routes/external/users.ts b/control-api/src/routes/external/users.ts
index 5378b7ca8..7f07f440b 100644
--- a/control-api/src/routes/external/users.ts
+++ b/control-api/src/routes/external/users.ts
@@ -107,7 +107,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {
const userId = (req as ExternalAuthedRequest).externalAuth?.userId
return userId ? `team_directory:${userId}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
@@ -285,7 +285,7 @@ export function createExternalUsersRouter(gateway: K8sGateway): Router {
const userId = (req as ExternalAuthedRequest).externalAuth?.userId
return userId ? `profile_password_update:${userId}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
try {
diff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.ts
index 007aa15ef..0faa9c705 100644
--- a/control-api/src/routes/gfs/grants.ts
+++ b/control-api/src/routes/gfs/grants.ts
@@ -896,7 +896,7 @@ export function registerGfsGrantRoutes(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
router.get('/gfs/grants', requireAuthForControlUI, grantsRateLimit, asyncHandler(handleGrantRead))
router.put(
diff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
index 4586ea55f..16daaf772 100644
--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts
+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
@@ -69,7 +69,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants-legacy:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
router.get(
'/gfs/grants/legacy-standalone',
diff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.ts
index e16c98697..0bdd0c325 100644
--- a/control-api/src/routes/gfs/shares.ts
+++ b/control-api/src/routes/gfs/shares.ts
@@ -45,7 +45,7 @@ export function registerGfsShareRoutes(router: Router): void {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsshares:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
router.get('/gfs/shares', requireAuthForControlUI, sharesRateLimit, asyncHandler(handleShareRead))
router.post(
diff --git a/control-api/src/routes/internal/oauth.ts b/control-api/src/routes/internal/oauth.ts
index f6b247c96..59e2f87e3 100644
--- a/control-api/src/routes/internal/oauth.ts
+++ b/control-api/src/routes/internal/oauth.ts
@@ -77,7 +77,7 @@ export function createInternalOAuthRouter(gateway: K8sGateway): Router {
: (req.internalService?.name ?? 'unknown')
return `mcp-oauth-internal:${uid}`
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
const recipeReader: RecipeReader = {
diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
index 21c56d2f6..931f43628 100644
--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
@@ -82,7 +82,7 @@ const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({
bucketType: 'plugin_workload_sdk_credential',
maxPerMinute: 120,
getBucketKey: pluginWorkloadSdkCredentialBucketKey,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
function requirePluginWorkloadSdkScope(req: Request, res: Response, next: NextFunction): void {
diff --git a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
index 80d3f72f5..dddf304cb 100644
--- a/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
+++ b/control-api/src/routes/mcp-host/renewal/refresh.routes.ts
@@ -31,7 +31,7 @@ export function createMcpHostRefreshRoutes(): Router {
const key = getMcpHostRefreshRateLimitKey(token)
return key ? `refresh:${key}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
index c84614eaf..d8d63a46d 100644
--- a/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
+++ b/control-api/src/routes/mcp-host/renewal/reissue.routes.ts
@@ -37,7 +37,7 @@ export function createMcpHostReissueRoutes(): Router {
const key = getMcpHostExpiredRefreshRateLimitKey(token)
return key ? `reissue:${key}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
index 763dc71e2..948d05d55 100644
--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
@@ -491,7 +491,7 @@ export function createUserApprovalRequestsRoutes(
if (!auth) return null
return mcpHostRateLimitBucketKey('recipe', auth)
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req, res, next) => {
void (async () => {
@@ -1389,7 +1389,7 @@ export function createUserApprovalRequestsRoutes(
const approvalId = String(req.params?.id || '').trim()
return approvalId ? `provider-decision:${approvalId}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
(req, res, next) => {
void (async () => {
diff --git a/control-api/src/routes/mcpOauth.ts b/control-api/src/routes/mcpOauth.ts
index 909d72b13..d7368b0e6 100644
--- a/control-api/src/routes/mcpOauth.ts
+++ b/control-api/src/routes/mcpOauth.ts
@@ -378,7 +378,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {
const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefined
return claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined
@@ -543,7 +543,7 @@ export function createMcpOauthRouter(gateway: K8sGateway): Router {
const claims = req.res?.locals?.mcpHostControl as McpHostControlClaims | undefined
return claims ? `mcp-oauth:${claims.sub}` : 'mcp-oauth:unknown'
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
const claims = res.locals.mcpHostControl as McpHostControlClaims | undefined
diff --git a/control-api/src/routes/recipeOauth.ts b/control-api/src/routes/recipeOauth.ts
index 60fb8877d..dae4f1baa 100644
--- a/control-api/src/routes/recipeOauth.ts
+++ b/control-api/src/routes/recipeOauth.ts
@@ -81,7 +81,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth:${req.recipe.namespace}/${req.recipe.name}` : null,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
const { namespace: recipeNamespace, name: recipeName } = req.recipe!
@@ -190,7 +190,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth-user:${req.recipe.namespace}/${req.recipe.name}` : null,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
// [SEC-5] recipe identity ONLY from the broker token sub; userId is a
@@ -305,7 +305,7 @@ export function createRecipeOauthRouter(gateway: K8sGateway): Router {
maxPerMinute: config.oauthBrokerRlPerMin,
getBucketKey: req =>
req.recipe ? `recipe-oauth-users:${req.recipe.namespace}/${req.recipe.name}` : null,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
// [SEC-6] recipe identity from the broker token sub; returns only THIS
diff --git a/control-api/src/routes/registry.ts b/control-api/src/routes/registry.ts
index a698a5dd1..f64ea28cb 100644
--- a/control-api/src/routes/registry.ts
+++ b/control-api/src/routes/registry.ts
@@ -32,7 +32,7 @@ export function createRegistryRouter(): Router {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `voucher:${sub}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req: UiAuthedRequest, res, next) => {
try {
diff --git a/control-api/src/routes/rpc-access/hosts.ts b/control-api/src/routes/rpc-access/hosts.ts
index 92b2b04be..e355357c0 100644
--- a/control-api/src/routes/rpc-access/hosts.ts
+++ b/control-api/src/routes/rpc-access/hosts.ts
@@ -61,7 +61,7 @@ export function createRpcAccessHostsRouter(gateway: K8sGateway): Router {
const hostRef = String(req.params?.hostRef || '').trim()
return hostRef ? `host-wake:${hostRef}` : null
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
async (req, res, next) => {
const hostRef = String(req.params.hostRef || '').trim()
diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.ts
index 42a04e36f..8480d1980 100644
--- a/control-api/src/routes/workflows/shared/rateLimit.ts
+++ b/control-api/src/routes/workflows/shared/rateLimit.ts
@@ -152,7 +152,7 @@ function adminCodexReadRateLimit() {
bucketType: 'admin_codex_read',
maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -161,7 +161,7 @@ function adminCodexWriteRateLimit() {
bucketType: 'admin_codex_write',
maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -196,7 +196,7 @@ export function codexOAuthCallbackRateLimits() {
bucketType: 'codex_oauth_callback',
maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,
getBucketKey: codexOAuthCallbackBucketKey,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
] as const
}
@@ -234,7 +234,7 @@ export function llmProviderAttemptAuthorizeRateLimits() {
bucketType: 'llm_provider_attempt_authorize',
maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,
getBucketKey: mcpHostAttemptRateLimitKey,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
}),
] as const
}
@@ -255,7 +255,7 @@ export function adminWorkflowTriggerRateLimit() {
bucketType: 'workflow_trigger',
maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,
getBucketKey: adminWorkflowTriggerRateLimitKey,
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -273,7 +273,7 @@ function workflowTriggerRateLimitFor(getCredential: (req: Request) => string | n
if (!credential) return null
return hashedWorkflowTriggerBucket(credential)
},
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -332,7 +332,7 @@ function workflowAdminReadRateLimit() {
bucketType: 'workflow_admin_read',
maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -341,7 +341,7 @@ function adminOutputsReadRateLimit() {
bucketType: 'admin_outputs_read',
maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -350,7 +350,7 @@ export function workflowGrantReadRateLimit() {
bucketType: 'workflow_grants_read',
maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
@@ -359,7 +359,7 @@ export function workflowGrantWriteRateLimit() {
bucketType: 'workflow_grants_write',
maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,
getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),
- onBackendUnavailable: 'open',
+ onBackendUnavailable: 'process-memory',
})
}
diff --git a/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts b/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts
index e234d4568..d5a3fb35f 100644
--- a/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts
+++ b/control-api/test/gfsLegacyStandaloneGrantsRateLimit.test.ts
@@ -45,6 +45,7 @@ const RESOURCE_ID = '20000000-0000-4000-8000-000000000001'
function allowed(remaining = 29) {
return {
allowed: true,
+ backendAvailable: true,
remaining,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -55,6 +56,7 @@ function allowed(remaining = 29) {
function denied() {
return {
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/middleware.processMemoryRateLimiter.test.ts b/control-api/test/middleware.processMemoryRateLimiter.test.ts
new file mode 100644
index 000000000..0cd45b10c
--- /dev/null
+++ b/control-api/test/middleware.processMemoryRateLimiter.test.ts
@@ -0,0 +1,60 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+ PROCESS_MEMORY_MAX_TRACKED_KEYS,
+ ProcessMemoryRateLimiter,
+} from '../src/middleware/processMemoryRateLimiter.js'
+
+const START_MS = 1_800_000_000_000
+
+describe('ProcessMemoryRateLimiter', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('allows maxPerMinute hits per key in a 60 s window that starts at the first hit', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(START_MS)
+ const limiter = new ProcessMemoryRateLimiter(2)
+
+ const first = await limiter.hit('a')
+ vi.setSystemTime(START_MS + 10_000)
+ const second = await limiter.hit('a')
+ const third = await limiter.hit('a')
+ const otherKey = await limiter.hit('b')
+ vi.setSystemTime(START_MS + 60_000)
+ const nextWindow = await limiter.hit('a')
+
+ expect(first).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 60_000 })
+ expect(second).toEqual({ outcome: 'allowed', totalHits: 2, resetMs: START_MS + 60_000 })
+ expect(third).toEqual({ outcome: 'denied', totalHits: 3, resetMs: START_MS + 60_000 })
+ expect(otherKey).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 70_000 })
+ expect(nextWindow).toEqual({ outcome: 'allowed', totalHits: 1, resetMs: START_MS + 120_000 })
+ })
+
+ it('refuses a new key once maxTrackedKeys keys are tracked in the window, and accepts new keys in the next window', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(START_MS)
+ const limiter = new ProcessMemoryRateLimiter(5, 2)
+
+ expect((await limiter.hit('a')).outcome).toBe('allowed')
+ expect((await limiter.hit('b')).outcome).toBe('allowed')
+ expect(await limiter.hit('c')).toEqual({ outcome: 'key_cap_reached' })
+ // Witness: a tracked key is still counted while the cap refuses new ones.
+ expect(await limiter.hit('a')).toMatchObject({ outcome: 'allowed', totalHits: 2 })
+
+ vi.setSystemTime(START_MS + 60_000)
+ expect(await limiter.hit('c')).toMatchObject({ outcome: 'allowed', totalHits: 1 })
+ })
+
+ it('tracks at most 100 000 keys by default', async () => {
+ expect(PROCESS_MEMORY_MAX_TRACKED_KEYS).toBe(100_000)
+ const limiter = new ProcessMemoryRateLimiter(1)
+ for (let n = 0; n < 100_000; n += 1) {
+ const decision = await limiter.hit(`key:${n}`)
+ if (decision.outcome !== 'allowed') throw new Error(`key:${n}: ${decision.outcome}`)
+ }
+ expect(await limiter.hit('key:100000')).toEqual({ outcome: 'key_cap_reached' })
+ // Witness: the counter was live for the keys under the cap.
+ expect(await limiter.hit('key:0')).toMatchObject({ outcome: 'denied', totalHits: 2 })
+ })
+})
diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.ts
index 58c936f5a..03f1b68e6 100644
--- a/control-api/test/middleware.rateLimitMiddleware.test.ts
+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
+import { MemoryStore } from 'express-rate-limit'
import { createHash } from 'node:crypto'
import request from 'supertest'
import type { RateLimitCheck } from '../src/services/rateLimiterService.js'
@@ -41,13 +42,30 @@ function unavailable(): RateLimitCheck {
}
}
-function appWith(onBackendUnavailable: 'open' | 'closed') {
+// Compile-time check only. control-api's tsconfig covers src/, so vitest does
+// not typecheck this file; run tsc over it to verify the expectation below.
+export function buildWithRemovedOpenMode() {
+ return rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'unit:key',
+ // @ts-expect-error 'open' is no longer a mode
+ onBackendUnavailable: 'open',
+ })
+}
+
+function appWith(onBackendUnavailable: 'process-memory' | 'closed') {
const handler = vi.fn((_req: express.Request, res: express.Response) => {
res.status(204).end()
})
+ const warn = vi.fn()
const app = express()
app.get(
'/limited',
+ (req, _res, next) => {
+ ;(req as unknown as { log: { warn: typeof warn } }).log = { warn }
+ next()
+ },
rateLimitMiddleware({
bucketType: 'unit_bucket',
maxPerMinute: LIMIT,
@@ -56,7 +74,7 @@ function appWith(onBackendUnavailable: 'open' | 'closed') {
}),
handler
)
- return { app, handler }
+ return { app, handler, warn }
}
describe('rateLimitMiddleware backend-unavailable policy', () => {
@@ -67,11 +85,15 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {
checkAndIncrement.mockResolvedValueOnce(unavailable())
- const { app, handler } = appWith('closed')
+ const { app, handler, warn } = appWith('closed')
const response = await request(app).get('/limited')
expect(response.status).toBe(503)
+ expect(warn).toHaveBeenCalledExactlyOnceWith(
+ { event: 'rate_limit_unavailable', bucketType: 'unit_bucket', suppressed: 0 },
+ 'rate limit backend unavailable'
+ )
expect(response.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
expect(response.headers['retry-after']).toBe('2')
expect(response.headers['cache-control']).toBe('no-store')
@@ -84,21 +106,175 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
expect(handler).not.toHaveBeenCalled()
})
- it("'open' lets the uncounted request through, as before", async () => {
- checkAndIncrement.mockResolvedValueOnce(unavailable())
- const { app, handler } = appWith('open')
+ it("'closed' writes one warn line per minute while the backend is down, and counts every 503", async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(1_800_000_000_000)
+ checkAndIncrement.mockResolvedValue(unavailable())
+ const { app, handler, warn } = appWith('closed')
- const response = await request(app).get('/limited')
+ const statuses: number[] = []
+ for (let n = 0; n < 50; n += 1) statuses.push((await request(app).get('/limited')).status)
- expect(response.status).toBe(204)
- expect(handler).toHaveBeenCalledTimes(1)
- expect(checkAndIncrement).toHaveBeenCalledExactlyOnceWith('unit:key', LIMIT)
- expect(hits.inc).toHaveBeenCalledExactlyOnceWith(
- { bucket_type: 'unit_bucket', result: 'allowed' },
- 1
+ // Witness: all 50 reached the limiter and were refused.
+ expect(statuses.filter(status => status === 503)).toHaveLength(50)
+ expect(checkAndIncrement).toHaveBeenCalledTimes(50)
+ expect(hits.inc).toHaveBeenCalledTimes(50)
+ expect(warn).toHaveBeenCalledTimes(1)
+
+ vi.setSystemTime(1_800_000_060_000)
+ expect((await request(app).get('/limited')).status).toBe(503)
+ expect(warn).toHaveBeenCalledTimes(2)
+ expect(warn.mock.calls[1]![0]).toMatchObject({
+ event: 'rate_limit_unavailable',
+ suppressed: 49,
+ })
+ expect(handler).not.toHaveBeenCalled()
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it("'process-memory' counts in memory while the backend cannot count, and answers 429 over the limit", async () => {
+ checkAndIncrement.mockResolvedValue(unavailable())
+ const { app, handler, warn } = appWith('process-memory')
+
+ const responses: request.Response[] = []
+ for (let n = 0; n <= LIMIT; n += 1) responses.push(await request(app).get('/limited'))
+
+ // Witness: every request reached the Postgres limiter first.
+ expect(checkAndIncrement).toHaveBeenCalledTimes(LIMIT + 1)
+ expect(responses.slice(0, LIMIT).map(response => response.status)).toEqual(
+ Array.from({ length: LIMIT }, () => 204)
)
+ expect(
+ responses.slice(0, LIMIT).map(response => response.headers['x-ratelimit-remaining'])
+ ).toEqual(['4', '3', '2', '1', '0'])
+ expect(handler).toHaveBeenCalledTimes(LIMIT)
+
+ const denied = responses[LIMIT]!
+ expect(denied.status).toBe(429)
+ const retryAfterSeconds = Number(denied.headers['retry-after'])
+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
+ expect(retryAfterSeconds).toBeLessThanOrEqual(60)
+ expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })
+ expect(denied.headers['x-ratelimit-limit']).toBe(String(LIMIT))
+ expect(denied.headers['x-ratelimit-remaining']).toBe('0')
+ expect(Number(denied.headers['x-ratelimit-reset'])).toBeGreaterThan(Date.now() / 1000)
+
+ expect(warn).toHaveBeenCalledExactlyOnceWith(
+ {
+ event: 'rate_limit_denied',
+ bucketType: 'unit_bucket',
+ hashedKey: createHash('sha256').update('unit:key').digest('hex'),
+ count: LIMIT + 1,
+ maxPerMinute: LIMIT,
+ source: 'process-memory',
+ },
+ 'rate limit exceeded'
+ )
+ expect(hits.inc.mock.calls).toEqual([
+ ...Array.from({ length: LIMIT }, () => [
+ { bucket_type: 'unit_bucket', result: 'fallback_allowed' },
+ 1,
+ ]),
+ [{ bucket_type: 'unit_bucket', result: 'fallback_denied' }, 1],
+ ])
+ })
+
+ it("'process-memory' leaves the decision to Postgres again once the backend can count", async () => {
+ const increment = vi.spyOn(MemoryStore.prototype, 'increment')
+ try {
+ for (let n = 0; n <= LIMIT; n += 1) checkAndIncrement.mockResolvedValueOnce(unavailable())
+ checkAndIncrement.mockResolvedValueOnce(available(1))
+ const { app, handler } = appWith('process-memory')
+
+ const statuses: number[] = []
+ for (let n = 0; n <= LIMIT + 1; n += 1)
+ statuses.push((await request(app).get('/limited')).status)
+
+ // The in-memory counter is exhausted, yet the counted request passes:
+ // Postgres decided it, and the in-memory store was not consulted.
+ expect(statuses).toEqual([...Array.from({ length: LIMIT }, () => 204), 429, 204])
+ expect(increment).toHaveBeenCalledTimes(LIMIT + 1)
+ // Liveness witness: the last request reached the Postgres limiter and the handler.
+ expect(checkAndIncrement).toHaveBeenCalledTimes(LIMIT + 2)
+ expect(handler).toHaveBeenCalledTimes(LIMIT + 1)
+ expect(hits.inc.mock.calls.at(-1)).toEqual([
+ { bucket_type: 'unit_bucket', result: 'allowed' },
+ 1,
+ ])
+ } finally {
+ increment.mockRestore()
+ }
})
+ it("'process-memory' answers 503 for a new key once 100 000 keys are tracked, and still counts tracked keys", async () => {
+ checkAndIncrement.mockResolvedValue(unavailable())
+ const middleware = rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: req => req.headers['x-key'] as string,
+ onBackendUnavailable: 'process-memory',
+ })
+ // Driven without HTTP: 100 001 supertest round trips would take minutes.
+ type Outcome = {
+ next: boolean
+ status?: number
+ body?: unknown
+ headers: Record<string, string>
+ }
+ const call = (key: string) =>
+ new Promise<Outcome>((resolve, reject) => {
+ const headers: Record<string, string> = {}
+ const res = {
+ statusCode: 200,
+ setHeader(name: string, value: string) {
+ headers[name.toLowerCase()] = value
+ },
+ status(code: number) {
+ this.statusCode = code
+ return this
+ },
+ json(body: unknown) {
+ resolve({ next: false, status: this.statusCode, body, headers })
+ },
+ }
+ const req = { headers: { 'x-key': key } }
+ middleware(req as never, res as never, (err?: unknown) =>
+ err ? reject(err) : resolve({ next: true, headers })
+ )
+ })
+
+ for (let n = 0; n < 100_000; n += 1) {
+ const outcome = await call(`key:${n}`)
+ if (!outcome.next) throw new Error(`key:${n} was not admitted: ${JSON.stringify(outcome)}`)
+ }
+ const overCap = await call('key:100000')
+ const tracked = await call('key:0')
+
+ expect(overCap).toEqual({
+ next: false,
+ status: 503,
+ body: { error: 'rate_limit_unavailable', retryAfterSeconds: 2 },
+ headers: { 'retry-after': '2', 'cache-control': 'no-store' },
+ })
+ // Witness: a key already tracked is still counted in memory (its second hit).
+ expect(tracked).toEqual({
+ next: true,
+ headers: expect.objectContaining({ 'x-ratelimit-remaining': String(LIMIT - 2) }),
+ })
+ expect(checkAndIncrement).toHaveBeenCalledTimes(100_002)
+ expect(hits.inc.mock.calls.at(-2)).toEqual([
+ { bucket_type: 'unit_bucket', result: 'unavailable' },
+ 1,
+ ])
+ expect(hits.inc.mock.calls.at(-1)).toEqual([
+ { bucket_type: 'unit_bucket', result: 'fallback_allowed' },
+ 1,
+ ])
+ }, 60_000)
+
it("'closed' still admits a counted request and still answers 429 over the limit", async () => {
checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))
const { app, handler } = appWith('closed')
diff --git a/control-api/test/recipeOauth.userToken.test.ts b/control-api/test/recipeOauth.userToken.test.ts
index b979c9bc9..b143f719d 100644
--- a/control-api/test/recipeOauth.userToken.test.ts
+++ b/control-api/test/recipeOauth.userToken.test.ts
@@ -26,6 +26,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -196,6 +197,7 @@ describe('routes/recipe-oauth — POST /recipe-oauth/user-token (SEC-5)', () =>
seedRecipe(gateway, { name: 'leadforge', backgroundAccess: true })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/recipeOauth.users.test.ts b/control-api/test/recipeOauth.users.test.ts
index e7cae7169..096a4410a 100644
--- a/control-api/test/recipeOauth.users.test.ts
+++ b/control-api/test/recipeOauth.users.test.ts
@@ -24,6 +24,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -138,8 +139,7 @@ describe('routes/recipe-oauth — GET /recipe-oauth/users (SEC-6)', () => {
})
it('401 without a broker token', async () => {
- const res = await request(app)
- .get('/api/v1/recipe-oauth/users?oauthClientId=google-gmail')
+ const res = await request(app).get('/api/v1/recipe-oauth/users?oauthClientId=google-gmail')
expect(res.status).toBe(401)
})
@@ -169,6 +169,7 @@ describe('routes/recipe-oauth — GET /recipe-oauth/users (SEC-6)', () => {
seedRecipe(gateway, { name: 'leadforge', backgroundAccess: true })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts
new file mode 100644
index 000000000..f22ec20f8
--- /dev/null
+++ b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts
@@ -0,0 +1,222 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import { config } from '../src/config.js'
+
+/**
+ * The public control-admin token routes (password reset, invitation, email
+ * confirmation) are called before login. Each route has two buckets:
+ *
+ * - IP + submitted value (email, login or token prefix) at
+ * `adminPublicTokenRlPerMin`: one person retrying the same value.
+ * - IP alone at `adminPublicTokenIpRlPerMin`: the ceiling on rotating values
+ * from one source. Many legitimate admins can share one public IP (a
+ * corporate VPN egress, an office NAT), so this ceiling is several times the
+ * per-value one, and a request the per-value bucket refuses is never charged
+ * to the IP bucket: one person's retry loop cannot use up the office's budget.
+ *
+ * The counter is an in-memory stand-in for the Postgres bucket; the real
+ * rateLimitMiddleware runs on top of it.
+ */
+
+const counts = vi.hoisted(() => new Map<string, number>())
+const limiterCalls = vi.hoisted(() => [] as Array<{ key: string; max: number }>)
+
+vi.mock('../src/services/rateLimiterService.js', () => ({
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,
+ checkAndIncrement: vi.fn(async (key: string, max: number) => {
+ limiterCalls.push({ key, max })
+ const count = (counts.get(key) ?? 0) + 1
+ counts.set(key, count)
+ return {
+ allowed: count <= max,
+ remaining: Math.max(0, max - count),
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count,
+ backendAvailable: true,
+ }
+ }),
+}))
+
+const registration = vi.hoisted(() => ({
+ registerAndSendControlAdminPasswordReset: vi.fn(),
+ validateControlAdminEmailConfirmationToken: vi.fn(),
+ validateControlAdminInvitationToken: vi.fn(),
+ validateControlAdminPasswordResetToken: vi.fn(),
+}))
+vi.mock('../src/services/controlAdminInvitationRegistrationService.js', () => registration)
+vi.mock('../src/services/directory/index.js', () => ({
+ acceptInvitationById: vi.fn(),
+ getPendingMemberInvitationForEmail: vi.fn(),
+ provisionAdminDesktopWorkspace: vi.fn(),
+ setInvitationPasswordForUser: vi.fn(),
+}))
+vi.mock('../src/services/adminAuthService.js', () => ({
+ completeControlAdminEmailChangeRequest: vi.fn(),
+ completeControlAdminInvitation: vi.fn(),
+ completeControlAdminPasswordResetRequest: vi.fn(),
+ createControlAdminPasswordResetRequest: vi.fn(),
+ findAdminById: vi.fn(),
+ findAdminByLogin: vi.fn(),
+ getPendingControlAdminEmailChangeRequest: vi.fn(),
+ getPendingControlAdminInvitation: vi.fn(),
+ getPendingControlAdminPasswordResetRequest: vi.fn(),
+ isValidAdminEmail: vi.fn(() => true),
+ isValidAdminUsername: vi.fn(() => true),
+ markControlAdminInvitationOpened: vi.fn(),
+ registerAdminFailedLogin: vi.fn(),
+ registerAdminSuccessfulLogin: vi.fn(),
+ revokeAdminTokenJti: vi.fn(),
+ revokeControlAdminPasswordResetRequest: vi.fn(),
+ setupInitialAdminCredentials: vi.fn(),
+}))
+vi.mock('../src/services/initialAdminSetupService.js', () => ({
+ setupInitialAdminWithDesktopWorkspace: vi.fn(),
+}))
+vi.mock('../src/utils/auth/adminAuthToken.js', () => ({ signAdminToken: vi.fn() }))
+vi.mock('../src/middleware/controlUIAuth.js', () => ({
+ requireAuthForControlUI: (_req: unknown, _res: unknown, next: () => void) => next(),
+}))
+
+const { createAdminAuthRouter } = await import('../src/routes/admin/auth.js')
+
+const PER_VALUE = 3
+const PER_IP = 10
+const VALIDATE = '/admin/auth/password-reset/validate'
+
+const PUBLIC_TOKEN_ROUTES = [
+ ['password_reset_request', '/admin/auth/password-reset/request'],
+ ['password_reset_validate', '/admin/auth/password-reset/validate'],
+ ['password_reset_complete', '/admin/auth/password-reset/complete'],
+ ['invitation_validate', '/admin/auth/control-admin-invitations/validate'],
+ ['invitation_complete', '/admin/auth/control-admin-invitations/complete'],
+ ['email_confirmation_validate', '/admin/auth/control-admin-email-confirmations/validate'],
+ ['email_confirmation_complete', '/admin/auth/control-admin-email-confirmations/complete'],
+] as const
+
+function buildApp() {
+ const warn = vi.fn()
+ const app = express()
+ app.set('trust proxy', 1)
+ app.use(express.json())
+ app.use((req, _res, next) => {
+ ;(req as unknown as { log: { warn: typeof warn } }).log = { warn }
+ next()
+ })
+ app.use(createAdminAuthRouter())
+ return { app, warn }
+}
+
+function validateFrom(app: express.Express, ip: string, email: string) {
+ return request(app)
+ .post(VALIDATE)
+ .set('x-forwarded-for', ip)
+ .send({ token: `token-${email}`, email })
+}
+
+function deniedBucketTypes(warn: ReturnType<typeof vi.fn>): string[] {
+ return warn.mock.calls
+ .map(([fields]) => fields as { event?: string; bucketType?: string })
+ .filter(fields => fields.event === 'rate_limit_denied')
+ .map(fields => fields.bucketType ?? '')
+}
+
+describe('public control-admin token limiter', () => {
+ beforeEach(() => {
+ counts.clear()
+ limiterCalls.length = 0
+ Object.values(registration).forEach(fn => fn.mockReset())
+ // The handler's own outcome is irrelevant here; reaching it is the witness.
+ registration.validateControlAdminPasswordResetToken.mockRejectedValue(new Error('invalid'))
+ config.adminPublicTokenRlPerMin = PER_VALUE
+ config.adminPublicTokenIpRlPerMin = PER_IP
+ })
+
+ it('lets every admin behind one shared public IP through, each with their own value', async () => {
+ const { app, warn } = buildApp()
+ const statuses: number[] = []
+ for (let user = 0; user < PER_IP; user += 1) {
+ statuses.push((await validateFrom(app, '203.0.113.7', `admin${user}@corp.test`)).status)
+ }
+
+ // More admins than the per-value limit, one request each, all from one IP.
+ expect(statuses).toHaveLength(PER_IP)
+ expect(statuses.every(status => status === 400)).toBe(true)
+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP)
+ expect(deniedBucketTypes(warn)).toEqual([])
+ })
+
+ it('refuses value rotation from one IP past the IP ceiling, and names the IP bucket', async () => {
+ const { app, warn } = buildApp()
+ for (let n = 0; n < PER_IP; n += 1) {
+ expect((await validateFrom(app, '203.0.113.7', `guess${n}@corp.test`)).status).toBe(400)
+ }
+
+ const refused = await validateFrom(app, '203.0.113.7', `guess${PER_IP}@corp.test`)
+ expect(refused.status).toBe(429)
+ expect(refused.headers['x-ratelimit-limit']).toBe(String(PER_IP))
+ expect(deniedBucketTypes(warn)).toEqual(['control_admin_public_ip_password_reset_validate'])
+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP)
+
+ // Another source is unaffected.
+ expect((await validateFrom(app, '198.51.100.9', '[email protected]')).status).toBe(400)
+ expect(registration.validateControlAdminPasswordResetToken).toHaveBeenCalledTimes(PER_IP + 1)
+ })
+
+ it('does not charge the IP bucket for a request the per-value bucket refused', async () => {
+ const { app, warn } = buildApp()
+ const retries: number[] = []
+ for (let attempt = 0; attempt < PER_VALUE + 5; attempt += 1) {
+ retries.push((await validateFrom(app, '203.0.113.7', '[email protected]')).status)
+ }
+ expect(retries.filter(status => status === 400)).toHaveLength(PER_VALUE)
+ expect(retries.filter(status => status === 429)).toHaveLength(5)
+ expect(new Set(deniedBucketTypes(warn))).toEqual(
+ new Set(['control_admin_public_password_reset_validate'])
+ )
+
+ // The IP bucket saw only the PER_VALUE admitted requests, so the other
+ // admins on that IP still have PER_IP - PER_VALUE requests.
+ const others: number[] = []
+ for (let user = 0; user < PER_IP - PER_VALUE; user += 1) {
+ others.push((await validateFrom(app, '203.0.113.7', `colleague${user}@corp.test`)).status)
+ }
+ expect(others).toHaveLength(PER_IP - PER_VALUE)
+ expect(others.every(status => status === 400)).toBe(true)
+ expect((await validateFrom(app, '203.0.113.7', '[email protected]')).status).toBe(429)
+ })
+
+ it('groups IPv6 sources by /56, so rotating addresses inside one allocation does not evade the ceiling', async () => {
+ const { app } = buildApp()
+ for (let n = 0; n < PER_IP; n += 1) {
+ // Every address differs, and all of them sit in 2001:db8:0:0000::/56.
+ const status = (
+ await validateFrom(app, `2001:db8:0:${n.toString(16)}::${n + 1}`, `v6-${n}@corp.test`)
+ ).status
+ expect(status).toBe(400)
+ }
+ expect((await validateFrom(app, '2001:db8:0:ff::1', '[email protected]')).status).toBe(429)
+ // The next /56 is another source.
+ expect((await validateFrom(app, '2001:db8:0:100::1', '[email protected]')).status).toBe(400)
+ })
+
+ it.each(PUBLIC_TOKEN_ROUTES)(
+ '%s charges the per-value bucket, then the per-IP bucket, at the configured limits',
+ async (routeName, path) => {
+ const { app } = buildApp()
+ await request(app)
+ .post(path)
+ .set('x-forwarded-for', '203.0.113.7')
+ .send({ token: 'tok', email: '[email protected]', password: 'x'.repeat(12) })
+
+ expect(limiterCalls).toEqual([
+ {
+ key: `control-admin-public:${routeName}:203.0.113.7:[email protected]`,
+ max: PER_VALUE,
+ },
+ { key: `control-admin-public-ip:${routeName}:203.0.113.7`, max: PER_IP },
+ ])
+ }
+ )
+})
diff --git a/control-api/test/routes.adminPluginWorkloadSdk.test.ts b/control-api/test/routes.adminPluginWorkloadSdk.test.ts
index 6f66c9648..51f9f9aee 100644
--- a/control-api/test/routes.adminPluginWorkloadSdk.test.ts
+++ b/control-api/test/routes.adminPluginWorkloadSdk.test.ts
@@ -118,6 +118,7 @@ beforeEach(() => {
vi.mocked(checkAndIncrement).mockReset()
vi.mocked(checkAndIncrement).mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 119,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.adminRegistryGrants.test.ts b/control-api/test/routes.adminRegistryGrants.test.ts
index 960405ead..71975e810 100644
--- a/control-api/test/routes.adminRegistryGrants.test.ts
+++ b/control-api/test/routes.adminRegistryGrants.test.ts
@@ -80,6 +80,7 @@ vi.mock('../src/services/registryConnectionDb.js', async importOriginal => {
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn(async () => ({
allowed: true,
+ backendAvailable: true,
remaining: 29,
resetMs: Date.now() + 60000,
})),
@@ -113,6 +114,7 @@ beforeEach(() => {
connDb.isRegistryAuthActive.mockImplementation(async () => cfg.registryAuthEnabled)
vi.mocked(checkAndIncrement).mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 29,
resetMs: Date.now() + 60000,
} as never)
diff --git a/control-api/test/routes.adminRegistryKeys.test.ts b/control-api/test/routes.adminRegistryKeys.test.ts
index c56e1e9b3..6234a81cd 100644
--- a/control-api/test/routes.adminRegistryKeys.test.ts
+++ b/control-api/test/routes.adminRegistryKeys.test.ts
@@ -57,6 +57,7 @@ vi.mock('../src/services/registryConnectionDb.js', () => connDb)
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn(async () => ({
allowed: true,
+ backendAvailable: true,
remaining: 29,
resetMs: Date.now() + 60000,
})),
@@ -90,6 +91,7 @@ beforeEach(() => {
connDb.isRegistryAuthActive.mockImplementation(async () => cfg.registryAuthEnabled)
vi.mocked(checkAndIncrement).mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 29,
resetMs: Date.now() + 60000,
} as never)
@@ -188,6 +190,7 @@ describe('GET /admin/registry/keys', () => {
it('429 when the per-admin rate limit is exceeded', async () => {
vi.mocked(checkAndIncrement).mockResolvedValue({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 60000,
} as never)
diff --git a/control-api/test/routes.auth.codexSubscriptionCallback.test.ts b/control-api/test/routes.auth.codexSubscriptionCallback.test.ts
index 52ed15c3e..11811b4d4 100644
--- a/control-api/test/routes.auth.codexSubscriptionCallback.test.ts
+++ b/control-api/test/routes.auth.codexSubscriptionCallback.test.ts
@@ -24,6 +24,7 @@ vi.mock('../src/db.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 19,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.externalGfs.grantsLimiterWiring.test.ts b/control-api/test/routes.externalGfs.grantsLimiterWiring.test.ts
new file mode 100644
index 000000000..b233b0033
--- /dev/null
+++ b/control-api/test/routes.externalGfs.grantsLimiterWiring.test.ts
@@ -0,0 +1,109 @@
+import { describe, expect, it, vi } from 'vitest'
+import type { Router } from 'express'
+
+/**
+ * #764: the two actor buckets in front of the external grant and share routes
+ * refuse a request they cannot count. An HTTP test cannot observe this option,
+ * because the pre-resolution Postgres bucket fails closed first on the same
+ * pool, so the wiring is read from the router: which options each limiter was
+ * built with, and which routes mount it.
+ */
+
+type LimiterOptions = { bucketType: string; onBackendUnavailable: 'process-memory' | 'closed' }
+type TaggedLimiter = ((...a: unknown[]) => void) & { limiterOptions: LimiterOptions }
+
+const built = vi.hoisted(() => [] as LimiterOptions[])
+
+vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({
+ rateLimitMiddleware: (options: LimiterOptions) => {
+ built.push(options)
+ const limiter = ((_req: unknown, _res: unknown, next: () => void) => next()) as TaggedLimiter
+ limiter.limiterOptions = options
+ return limiter
+ },
+}))
+vi.mock('../src/config.js', () => ({
+ config: {
+ gfscBaseUrl: 'http://gfsc.gfs.svc:8087',
+ gfscWriteBaseUrl: 'http://gfsc-writer.gfs.svc:8087',
+ gfscProxyTimeoutMs: 300_000,
+ gfsUploadMaxPartBytes: 16 * 1024 * 1024,
+ hostsNamespace: 'mcp-host',
+ desktopGfsOperatorLinkingEnabled: false,
+ externalGfsIngressRlPerMin: 1800,
+ externalGfsTokenUserRlPerMin: 10,
+ externalGfsTokenIpRlPerMin: 600,
+ externalGfsIpRlPerMin: 1200,
+ externalGfsResourceReadRlPerMin: 120,
+ externalGfsProxyReadRlPerMin: 60,
+ externalGfsGrantsReadRlPerMin: 45,
+ externalGfsSharesReadRlPerMin: 35,
+ externalGfsOperationRlPerMin: 30,
+ },
+}))
+// Both derive signing keys at module load; building the router never signs.
+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ verifyExternalSessionToken: vi.fn(),
+}))
+vi.mock('../src/auth/gfsToken.js', () => ({
+ GFS_DELETE_SCOPE: 'gfs.delete',
+ GFS_READ_SCOPE: 'gfs.read',
+ GFS_WRITE_SCOPE: 'gfs.write',
+ GFS_SCOPES: ['gfs.read', 'gfs.write', 'gfs.delete', 'gfs.manage_acl', 'gfs.share'],
+ signGfsToken: vi.fn(),
+}))
+vi.mock('../src/db.js', () => ({
+ pool: { query: vi.fn() },
+ rateLimitPool: { query: vi.fn() },
+ withTransaction: vi.fn(),
+}))
+// grants.ts/shares.ts/token.ts import requireAuthForControlUI, which derives the
+// admin JWT key at module load; these routes never call it.
+vi.mock('../src/middleware/controlUIAuth.js', () => ({
+ requireAuthForControlUI: (_req: unknown, _res: unknown, next: () => void) => next(),
+}))
+
+const { createExternalGfsRouter } = await import('../src/routes/external/gfs.js')
+
+type RouteLayer = {
+ route?: { path: string; methods: Record<string, boolean>; stack: Array<{ handle: unknown }> }
+}
+
+function actorLimiterOf(router: Router, method: string, path: string): LimiterOptions {
+ const layer = (router.stack as RouteLayer[]).find(
+ l => l.route?.path === path && l.route.methods[method] === true
+ )
+ if (!layer?.route) throw new Error(`route ${method.toUpperCase()} ${path} is not mounted`)
+ const limiters = layer.route.stack
+ .map(s => (s.handle as Partial<TaggedLimiter>).limiterOptions)
+ .filter((o): o is LimiterOptions => o !== undefined)
+ expect(limiters).toHaveLength(1)
+ return limiters[0]!
+}
+
+describe('external GFS grant and share actor limiters', () => {
+ const router = createExternalGfsRouter()
+
+ it('builds both actor buckets to fail closed', () => {
+ // Witness: the router built exactly these two limiters, so the rows below
+ // are the options the routes use and not an empty match.
+ expect(built.map(o => o.bucketType).sort()).toEqual([
+ 'gfs_grants_external',
+ 'gfs_grants_external_read',
+ ])
+ for (const options of built) expect(options.onBackendUnavailable).toBe('closed')
+ })
+
+ it.each([
+ ['get', '/external/gfs/grants', 'gfs_grants_external_read'],
+ ['get', '/external/gfs/shares', 'gfs_grants_external_read'],
+ ['put', '/external/gfs/grants', 'gfs_grants_external'],
+ ['delete', '/external/gfs/grants/:id', 'gfs_grants_external'],
+ ['post', '/external/gfs/shares', 'gfs_grants_external'],
+ ['delete', '/external/gfs/shares/:id', 'gfs_grants_external'],
+ ])('%s %s mounts %s, which fails closed', (method, path, bucketType) => {
+ const limiter = actorLimiterOf(router, method, path)
+ expect(limiter.bucketType).toBe(bucketType)
+ expect(limiter.onBackendUnavailable).toBe('closed')
+ })
+})
diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
index 8c5bf1b53..8e11bfa78 100644
--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
@@ -261,7 +261,7 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
await corePool.query('DROP FUNCTION IF EXISTS t3b_limiter_fault()')
}
- it('T3b: the external grants limiter fails closed on a backend error while an open-policy route still passes', async () => {
+ it('T3b: the external grants limiter fails closed on a backend error while a process-memory route still passes', async () => {
const app = mod.createApp(new mod.MockGateway())
const userId = await seedUser()
const internalToken = mod.config.internalServiceTokens['external-rest-api']
@@ -292,8 +292,9 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
expect(grants.body).toEqual({ error: 'rate_limit_unavailable', retryAfterSeconds: 2 })
expect(grants.headers['retry-after']).toBe('2')
expect(grants.headers['cache-control']).toBe('no-store')
- // Witness for the open policy under the same fault: the ack route's
- // limiter could not count either, and its handler still answered.
+ // Witness for the process-memory policy under the same fault: the ack
+ // route's Postgres limiter could not count either, its in-memory counter
+ // admitted the first request, and its handler answered.
expect(ack.status).toBe(404)
expect(ack.body).toEqual({ error: 'notification_not_found' })
diff --git a/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts b/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts
index b1b430f3c..245de369a 100644
--- a/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts
+++ b/control-api/test/routes.internal.mcpOauthAuthorizeUrl.test.ts
@@ -27,6 +27,7 @@ vi.mock('../src/db.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -281,6 +282,7 @@ describe('POST /api/v1/internal/mcp-oauth/authorize-url (U5)', () => {
seedOauthServer(gateway, { name: 'gdrive', grantScope: 'user' })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts b/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts
index 3522bb192..3a77b9a12 100644
--- a/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts
+++ b/control-api/test/routes.internal.mcpOauthDisconnect.audit.test.ts
@@ -28,6 +28,7 @@ vi.mock('../src/db.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.internal.mcpOauthDisconnect.test.ts b/control-api/test/routes.internal.mcpOauthDisconnect.test.ts
index e9b43a1b9..44552a2d2 100644
--- a/control-api/test/routes.internal.mcpOauthDisconnect.test.ts
+++ b/control-api/test/routes.internal.mcpOauthDisconnect.test.ts
@@ -45,6 +45,7 @@ vi.mock('../src/db.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -258,6 +259,7 @@ describe('DELETE /api/v1/internal/mcp-oauth/grant (spec 11 U4)', () => {
seedOauthServer(gateway, { name: 'gdrive' })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts
index 99653a8a3..4100f5574 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.attestation.test.ts
@@ -10,6 +10,7 @@ import * as mcpHostJwt from '../src/utils/auth/mcpHostJwtToken.js'
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts
index 667296841..36a0b11b7 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.createApp.test.ts
@@ -28,6 +28,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
index 066407bbd..9760e36a2 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
@@ -13,6 +13,7 @@ import * as mcpHostJwt from '../src/utils/auth/mcpHostJwtToken.js'
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.mcp-host.workflows.test.ts b/control-api/test/routes.mcp-host.workflows.test.ts
index 52fd88e43..5e9da9e38 100644
--- a/control-api/test/routes.mcp-host.workflows.test.ts
+++ b/control-api/test/routes.mcp-host.workflows.test.ts
@@ -61,6 +61,7 @@ vi.mock('../src/services/adminAuthService.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 10,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.mcpOauth.test.ts b/control-api/test/routes.mcpOauth.test.ts
index c6c4ba9f6..f84fcaffb 100644
--- a/control-api/test/routes.mcpOauth.test.ts
+++ b/control-api/test/routes.mcpOauth.test.ts
@@ -24,6 +24,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -608,6 +609,7 @@ describe('routes/mcp-oauth — POST /mcp-oauth/grants/exists (mini-spec 13)', ()
seedOauthServer(gateway, { name: 'gdrive', grantScope: 'user' })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.profile.test.ts b/control-api/test/routes.profile.test.ts
index 042ea4596..ee5a3ac86 100644
--- a/control-api/test/routes.profile.test.ts
+++ b/control-api/test/routes.profile.test.ts
@@ -132,6 +132,7 @@ describe('routes/profile', () => {
rateLimitMock.checkAndIncrement.mockReset()
rateLimitMock.checkAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.ts b/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.ts
new file mode 100644
index 000000000..99ea343a0
--- /dev/null
+++ b/control-api/test/routes.rateLimitProcessMemory.realPostgres.integration.test.ts
@@ -0,0 +1,231 @@
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { randomBytes, randomUUID } from 'node:crypto'
+import { Pool, type PoolClient } from 'pg'
+import request from 'supertest'
+
+// rateLimitMiddleware's 'process-memory' mode against real PostgreSQL: with
+// every limiter pool connection held, a real route's limiter cannot count in
+// Postgres and counts in process memory instead. Requests go through the
+// production Express app; the ONLY mock is the session-token verifier
+// (identity), as in routes.externalGfsRateLimitFailClosed. Skipped without
+// CONTROL_API_REAL_PG_ADMIN_URL.
+//
+// The limiter pool is small with a short acquire timeout, and the external
+// per-operation limit is low, so each held-pool request waits 200 ms instead
+// of the production 5 s and the suite needs four of them, not sixty-one.
+
+vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ // A token that is not a JSON identity fails verification, as a bad
+ // signature does in production.
+ verifyExternalSessionToken: (token: string) => {
+ if (!token.startsWith('{')) return null
+ const parsed = JSON.parse(token) as { userId: string }
+ return {
+ userId: parsed.userId,
+ email: `${parsed.userId}@example.test`,
+ teamId: null,
+ role: 'member',
+ authGeneration: 1,
+ exp: Math.floor(Date.now() / 1000) + 3600,
+ }
+ },
+}))
+
+const LIMITER_POOL_MAX = 2
+const LIMITER_ACQUIRE_TIMEOUT_MS = 200
+const EXTERNAL_PER_MIN = 3
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+describeRealPostgres('rateLimitMiddleware process-memory mode (real PostgreSQL)', () => {
+ const database = `control_api_rl_process_memory_${randomBytes(6).toString('hex')}`
+ const envKeys = [
+ 'CONTROL_API_PG_CONNECTION_STRING',
+ 'CORE_POOL_MAX',
+ 'CORE_POOL_CONNECTION_TIMEOUT_MS',
+ 'CORE_POOL_STATEMENT_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_MAX',
+ 'RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS',
+ 'RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS',
+ 'APPROVAL_RL_EXTERNAL_PER_MIN',
+ ] as const
+ const previousEnv = new Map<string, string | undefined>()
+
+ let adminPool: Pool
+ let corePool: Pool
+ let limiterPool: Pool
+ let mod: {
+ createApp: (gateway: unknown) => import('express').Express
+ config: typeof import('../src/config.js').config
+ hitsTotal: typeof import('../src/observability/metrics.js').rateLimitHitsTotal
+ MockGateway: typeof import('./mockGateway.js').MockGateway
+ }
+
+ beforeAll(async () => {
+ if (adminUrl === undefined) {
+ throw new Error('CONTROL_API_REAL_PG_ADMIN_URL is required by this suite')
+ }
+ const connectionString = databaseUrl(adminUrl, database)
+ for (const key of envKeys) {
+ previousEnv.set(key, process.env[key])
+ delete process.env[key]
+ }
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ process.env.CONTROL_API_PG_CONNECTION_STRING = connectionString
+ process.env.RATE_LIMIT_POOL_MAX = String(LIMITER_POOL_MAX)
+ process.env.RATE_LIMIT_POOL_CONNECTION_TIMEOUT_MS = String(LIMITER_ACQUIRE_TIMEOUT_MS)
+ process.env.APPROVAL_RL_EXTERNAL_PER_MIN = String(EXTERNAL_PER_MIN)
+
+ const dbMod = await import('../src/db.js')
+ corePool = dbMod.pool as unknown as Pool
+ corePool.on('error', () => {})
+ limiterPool = dbMod.rateLimitPool as unknown as Pool
+ limiterPool.on('error', () => {})
+ const migratePool = new Pool({ connectionString })
+ await dbMod.initDb({ connect: () => migratePool.connect() })
+ await migratePool.end()
+
+ const appMod = await import('../src/app.js')
+ const configMod = await import('../src/config.js')
+ const metricsMod = await import('../src/observability/metrics.js')
+ const { MockGateway } = await import('./mockGateway.js')
+ mod = {
+ createApp: appMod.createApp as never,
+ config: configMod.config,
+ hitsTotal: metricsMod.rateLimitHitsTotal,
+ MockGateway,
+ }
+ }, 60_000)
+
+ afterAll(async () => {
+ for (const key of envKeys) {
+ const value = previousEnv.get(key)
+ if (value === undefined) delete process.env[key]
+ else process.env[key] = value
+ }
+ await corePool?.end().catch(() => {})
+ await limiterPool?.end().catch(() => {})
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ async function seedUser(): Promise<string> {
+ const id = randomUUID()
+ await corePool.query(`INSERT INTO users (id, email, name) VALUES ($1, $2, $3)`, [
+ id,
+ `${id}@example.test`,
+ 'process-memory',
+ ])
+ return id
+ }
+
+ async function ack(app: import('express').Express, userId: string) {
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ return request(app)
+ .post(`/api/v1/external/notifications/${randomUUID()}/ack`)
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', JSON.stringify({ userId }))
+ .set('x-forwarded-for', '203.0.113.7')
+ }
+
+ async function externalUserHits(): Promise<Record<string, number>> {
+ const { values } = await mod.hitsTotal.get()
+ const totals: Record<string, number> = {}
+ for (const sample of values) {
+ if (sample.labels.bucket_type !== 'external_user') continue
+ const result = String(sample.labels.result)
+ totals[result] = (totals[result] ?? 0) + sample.value
+ }
+ return totals
+ }
+
+ function delta(after: Record<string, number>, before: Record<string, number>, result: string) {
+ return (after[result] ?? 0) - (before[result] ?? 0)
+ }
+
+ /** Read through the core pool: the limiter pool is the one being held. */
+ async function ledgerCount(bucketKey: string): Promise<number | null> {
+ const result = await corePool.query<{ count: number }>(
+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1',
+ [bucketKey]
+ )
+ return result.rows[0]?.count ?? null
+ }
+
+ it('T5: with every limiter connection held, the ack route admits max requests from process memory and answers 429 to the next; after release Postgres decides again', async () => {
+ expect(mod.config.approvalRlExternalPerMin).toBe(EXTERNAL_PER_MIN)
+ const app = mod.createApp(new mod.MockGateway())
+ const userId = await seedUser()
+ const bucketKey = `user:${userId}:notification-ack`
+ await corePool.query('DELETE FROM rate_limit_buckets')
+ const hitsBefore = await externalUserHits()
+
+ const held: PoolClient[] = []
+ const responses: request.Response[] = []
+ try {
+ for (let index = 0; index < LIMITER_POOL_MAX; index += 1) {
+ held.push(await limiterPool.connect())
+ }
+ // Witness: the pool is exhausted, not merely busy.
+ expect(limiterPool.totalCount).toBe(LIMITER_POOL_MAX)
+ expect(limiterPool.idleCount).toBe(0)
+
+ for (let n = 0; n <= EXTERNAL_PER_MIN; n += 1) responses.push(await ack(app, userId))
+ } finally {
+ for (const client of held) client.release()
+ }
+
+ // The handler ran for the first max requests: an unknown id is a 404.
+ for (const admitted of responses.slice(0, EXTERNAL_PER_MIN)) {
+ expect(admitted.status).toBe(404)
+ expect(admitted.body).toEqual({ error: 'notification_not_found' })
+ }
+ const denied = responses[EXTERNAL_PER_MIN]!
+ expect(denied.status).toBe(429)
+ const retryAfterSeconds = Number(denied.headers['retry-after'])
+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
+ expect(retryAfterSeconds).toBeLessThanOrEqual(60)
+ expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })
+ expect(denied.headers['x-ratelimit-remaining']).toBe('0')
+
+ const hitsHeld = await externalUserHits()
+ expect(delta(hitsHeld, hitsBefore, 'fallback_allowed')).toBe(EXTERNAL_PER_MIN)
+ expect(delta(hitsHeld, hitsBefore, 'fallback_denied')).toBe(1)
+ expect(delta(hitsHeld, hitsBefore, 'allowed')).toBe(0)
+ expect(await ledgerCount(bucketKey)).toBeNull()
+
+ // The in-memory counter for this key is over the limit, so an admitted
+ // request now proves Postgres decided it.
+ const recovered = await ack(app, userId)
+ expect(recovered.status).toBe(404)
+ expect(recovered.body).toEqual({ error: 'notification_not_found' })
+ expect(recovered.headers['x-ratelimit-remaining']).toBe(String(EXTERNAL_PER_MIN - 1))
+ // Liveness witness for the empty ledger above: the same request, once the
+ // pool is free, is counted in rate_limit_buckets.
+ expect(await ledgerCount(bucketKey)).toBe(1)
+ const hitsAfter = await externalUserHits()
+ expect(delta(hitsAfter, hitsHeld, 'allowed')).toBe(1)
+ expect(delta(hitsAfter, hitsHeld, 'fallback_allowed')).toBe(0)
+ expect(delta(hitsAfter, hitsHeld, 'fallback_denied')).toBe(0)
+ }, 30_000)
+})
diff --git a/control-api/test/routes.recipeOauth.test.ts b/control-api/test/routes.recipeOauth.test.ts
index 8c45d093f..06fcbdc44 100644
--- a/control-api/test/routes.recipeOauth.test.ts
+++ b/control-api/test/routes.recipeOauth.test.ts
@@ -26,6 +26,7 @@ vi.mock('../src/services/notificationEmitter.js', () => ({
vi.mock('../src/services/rateLimiterService.js', () => ({
checkAndIncrement: vi.fn().mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -202,6 +203,7 @@ describe('routes/recipe-oauth — POST /recipe-oauth/token', () => {
seedRecipe(gateway, { name: 'crm', backgroundAccess: true })
vi.mocked(checkAndIncrement).mockResolvedValueOnce({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30_000,
windowStartMs: Date.now(),
diff --git a/control-api/test/routes.registryConnect.test.ts b/control-api/test/routes.registryConnect.test.ts
index 4d1e870f1..07b681dad 100644
--- a/control-api/test/routes.registryConnect.test.ts
+++ b/control-api/test/routes.registryConnect.test.ts
@@ -117,6 +117,7 @@ beforeEach(() => {
cfg.registryUrl = 'https://example.com'
rateLimiter.checkAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60000,
})
@@ -521,6 +522,7 @@ describe('registry connect flow', () => {
it('GET 429s when the per-admin bucket is exceeded, without touching the registry', async () => {
rateLimiter.checkAndIncrement.mockResolvedValue({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30000,
})
@@ -558,6 +560,7 @@ describe('registry connect flow', () => {
connDb.getRegistryConnection.mockResolvedValue(null)
rateLimiter.checkAndIncrement.mockResolvedValue({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30000,
})
@@ -1031,6 +1034,7 @@ describe('POST recover', () => {
})
rateLimiter.checkAndIncrement.mockResolvedValue({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30000,
})
diff --git a/control-api/test/routes.rpcAccessHostsWake.test.ts b/control-api/test/routes.rpcAccessHostsWake.test.ts
index 0b610cf9c..94288cac0 100644
--- a/control-api/test/routes.rpcAccessHostsWake.test.ts
+++ b/control-api/test/routes.rpcAccessHostsWake.test.ts
@@ -81,6 +81,7 @@ describe('POST /rpc/hosts/:hostRef/wake', () => {
authMock.verifyRpcAccessToken.mockReturnValue(makeClaims())
rateLimiterMock.checkAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: 0,
@@ -128,6 +129,7 @@ describe('POST /rpc/hosts/:hostRef/wake', () => {
it('returns 429 with Retry-After when over the per-host limit, with no mutation', async () => {
rateLimiterMock.checkAndIncrement.mockResolvedValue({
allowed: false,
+ backendAvailable: true,
remaining: 0,
resetMs: Date.now() + 30_000,
windowStartMs: 0,
diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.ts
index 4d3003f6b..e05695b0f 100644
--- a/control-api/test/workflows.shared.rateLimit.test.ts
+++ b/control-api/test/workflows.shared.rateLimit.test.ts
@@ -60,6 +60,7 @@ function signedClaims(sub: string) {
function pgAllows() {
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -204,6 +205,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -433,6 +435,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -462,6 +465,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -492,6 +496,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -520,6 +525,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -550,6 +556,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 59,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -679,6 +686,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -718,6 +726,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -748,6 +757,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -779,6 +789,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),
@@ -818,6 +829,7 @@ describe('routes/workflows/shared/rateLimit', () => {
mockCheckAndIncrement.mockReset()
mockCheckAndIncrement.mockResolvedValue({
allowed: true,
+ backendAvailable: true,
remaining: 9,
resetMs: Date.now() + 60_000,
windowStartMs: Date.now(),