Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): close the remaining limiter gaps from the second review round

Alfredo Lopez7dc6e80fab66
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index f1b260b97..e82d0cdbe 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.618",
+ "version": "0.1.619",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.618",
+ "version": "0.1.619",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 2bee1119c..39a202743 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.618",
+ "version": "0.1.619",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 9a6e8de87..08b9a5abb 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -136,14 +136,14 @@ const DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = 3_000
export const RATE_LIMIT_POOL_SESSION_OPTIONS = '-c synchronous_commit=off'
/**
- * Unset (or empty) means the default. A set value outside `[min, max]`, or one
- * that is not canonical decimal integer text, stops startup instead of
- * silently becoming the default. Number() alone would also read '6.0', '0x6',
- * '6e0' and ' 6' as 6.
+ * Unset, empty or whitespace-only means the default, as in config.ts. A set
+ * value outside `[min, max]`, or one that is not canonical decimal integer
+ * text, stops startup instead of silently becoming the default. Number() alone
+ * would also read '6.0', '0x6', '6e0' and ' 6' as 6.
*/
function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {
const raw = process.env[name]
- if (!raw) return fallback
+ if (raw === undefined || raw.trim() === '') return fallback
const value = Number(raw)
if (!/^(0|[1-9]\d*)$/.test(raw) || value < min || value > max) {
throw new Error(`${name} must be an integer in [${min}, ${max}], got ${JSON.stringify(raw)}`)
diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.ts
index b836a1575..88510b723 100644
--- a/control-api/src/middleware/externalGfsRateLimit.ts
+++ b/control-api/src/middleware/externalGfsRateLimit.ts
@@ -379,6 +379,11 @@ async function enforceBuckets(input: {
latencyMs,
authorityResolutionAvoided: input.phase === 'pre-resolution',
})
+ // Earlier buckets that admitted this request wrote their counts; a 503
+ // charged no budget, so it carries none of them.
+ input.res.removeHeader('X-RateLimit-Limit')
+ input.res.removeHeader('X-RateLimit-Remaining')
+ input.res.removeHeader('X-RateLimit-Reset')
input.res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
input.res.setHeader('Cache-Control', 'no-store')
input.res.status(503).json({
diff --git a/control-api/src/middleware/externalSessionAuth.ts b/control-api/src/middleware/externalSessionAuth.ts
index 3cf4a859f..abf10ae32 100644
--- a/control-api/src/middleware/externalSessionAuth.ts
+++ b/control-api/src/middleware/externalSessionAuth.ts
@@ -13,15 +13,30 @@ export const SESSION_BACKEND_RETRY_AFTER_SECONDS = 2
// 57014 statement timeout), 58 system error.
const BACKEND_UNAVAILABLE_SQLSTATE_CLASSES = new Set(['08', '53', '57', '58'])
+// A Node system error code (ECONNREFUSED, ETIMEDOUT, EPIPE, ...). Node's own
+// argument and state errors use ERR_* codes, which this does not match.
+const SYSTEM_ERROR_CODE = /^E[A-Z]+$/
+
/**
* True when the lookup failed because PostgreSQL could not be reached or could
- * not run it. Pool acquire timeouts and socket failures are plain Errors with
- * no SQLSTATE. A DatabaseError in any other class (22P02, 42P01, ...) is a
- * defect in the query or the schema, which a 503 would hide.
+ * not run it:
+ * - a DatabaseError whose SQLSTATE class is 08, 53, 57 or 58;
+ * - a plain Error, which is how pg and pg-pool report an acquire timeout or a
+ * dropped connection;
+ * - an error carrying a Node system code, as socket failures do (including the
+ * AggregateError Node raises when every address of a host refuses).
+ * Anything else is a defect: a DatabaseError in another class (22P02, 42P01,
+ * ...) is in the query or the schema, and a TypeError or RangeError is in this
+ * process's code. A 503 would hide either one.
*/
function isBackendUnavailableError(error: unknown): boolean {
- if (!(error instanceof DatabaseError)) return true
- return BACKEND_UNAVAILABLE_SQLSTATE_CLASSES.has(String(error.code).slice(0, 2))
+ if (error instanceof DatabaseError) {
+ return BACKEND_UNAVAILABLE_SQLSTATE_CLASSES.has(String(error.code).slice(0, 2))
+ }
+ if (!(error instanceof Error)) return false
+ const code = (error as { code?: unknown }).code
+ if (typeof code === 'string' && SYSTEM_ERROR_CODE.test(code)) return true
+ return error.constructor === Error
}
export type ExternalAuthedRequest = Request & {
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.ts
index 943e36659..d7e285eb5 100644
--- a/control-api/src/middleware/gfsUploadAdmission.ts
+++ b/control-api/src/middleware/gfsUploadAdmission.ts
@@ -108,7 +108,11 @@ function retryAfterSeconds(check: RateLimitCheck): number {
function sendBackendUnavailable(res: Response): void {
res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
- res.status(503).json({ error: 'gfs_upload_admission_unavailable' })
+ res.setHeader('Cache-Control', 'no-store')
+ res.status(503).json({
+ error: 'gfs_upload_admission_unavailable',
+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
+ })
}
function sendRateLimited(res: Response, limit: string, retryAfter: number, max: number): void {
diff --git a/control-api/src/middleware/processMemoryRateLimiter.ts b/control-api/src/middleware/processMemoryRateLimiter.ts
index 86c69b992..5eb9d05bf 100644
--- a/control-api/src/middleware/processMemoryRateLimiter.ts
+++ b/control-api/src/middleware/processMemoryRateLimiter.ts
@@ -23,6 +23,12 @@ export type ProcessMemoryDecision =
* `maxTrackedKeys`. The store keeps a key until its cleanup timer has run
* twice without a hit on it, so it can hold a few times `maxTrackedKeys`
* entries at once; the cap bounds how fast that grows.
+ *
+ * The set's window is this class's own and does not line up with each key's
+ * window in the store. A key whose store window is still running is always
+ * admitted, so clearing the set cannot turn a key that is already being
+ * counted into a `key_cap_reached` refusal. Such a key was admitted under the
+ * cap when its window opened, so this does not raise the bound.
*/
export class ProcessMemoryRateLimiter {
private readonly store = new MemoryStore()
@@ -45,7 +51,12 @@ export class ProcessMemoryRateLimiter {
this.windowStartMs = now
}
if (!this.keysThisWindow.has(key)) {
- if (this.keysThisWindow.size >= this.maxTrackedKeys) return { outcome: 'key_cap_reached' }
+ if (
+ this.keysThisWindow.size >= this.maxTrackedKeys &&
+ !(await this.hasLiveWindow(key, now))
+ ) {
+ return { outcome: 'key_cap_reached' }
+ }
this.keysThisWindow.add(key)
}
@@ -59,4 +70,9 @@ export class ProcessMemoryRateLimiter {
resetMs: resetTime.getTime(),
}
}
+
+ private async hasLiveWindow(key: string, now: number): Promise<boolean> {
+ const client = await this.store.get(key)
+ return client?.resetTime !== undefined && client.resetTime.getTime() > now
+ }
}
diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.ts
index 0bcd39d94..fca6bc1c3 100644
--- a/control-api/src/middleware/rateLimitMiddleware.ts
+++ b/control-api/src/middleware/rateLimitMiddleware.ts
@@ -2,12 +2,37 @@ import { NextFunction, Request, Response } from 'express'
import { createHash } from 'node:crypto'
import { LogThrottle } from '../observability/logThrottle.js'
import { rateLimitHitsTotal } from '../observability/metrics.js'
+import { boundedBucketKey } from '../services/rateLimitBucketKey.js'
import {
RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,
checkAndIncrement,
} from '../services/rateLimiterService.js'
import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'
+/** What a limiter does when its Postgres backend cannot count a request. */
+export type RateLimitBackendUnavailableMode = 'process-memory' | 'closed'
+
+// Compile-time guard, checked by the CI typecheck of src/. The `'open'` mode
+// (let the request through uncounted) was removed; adding it back to the union
+// makes this alias fail to compile (TS2344).
+type AssertTrue<T extends true> = T
+export type OpenIsNotABackendUnavailableMode = AssertTrue<
+ 'open' extends RateLimitBackendUnavailableMode ? false : true
+>
+
+export type RateLimitEnforcerOptions = {
+ bucketType: string
+ maxPerMinute: number
+ onBackendUnavailable: RateLimitBackendUnavailableMode
+}
+
+/**
+ * Charges one request to `key` and answers it when it must not proceed.
+ * Resolves `true` when the caller may continue (the `X-RateLimit-*` headers are
+ * set), or `false` after a 429 or 503 has been sent.
+ */
+export type RateLimitEnforcer = (req: Request, res: Response, key: string) => Promise<boolean>
+
/**
* Factory for a per-request rate limit middleware backed by a PG token bucket.
*
@@ -31,6 +56,9 @@ import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'
* - If `getBucketKey` returns `null` (e.g. unable to derive key), the
* request is *allowed* but not counted — fail-open. Callers that need
* strict enforcement should return a sentinel key instead.
+ * - A key longer than 512 UTF-8 bytes is counted under its SHA-256
+ * (`boundedBucketKey`), so a client-supplied value cannot grow either
+ * backend's storage per request.
* - When the limiter backend cannot count the request
* (`backendAvailable: false`: the limiter pool is saturated, Postgres
* returned an error, or the upsert returned no row), `onBackendUnavailable`
@@ -41,19 +69,49 @@ import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'
* key. Over the limit it answers 429 like a Postgres denial, and the
* warn line carries `source: 'process-memory'`. The in-memory counter
* is used only for requests Postgres could not count; the two counts
- * are never added together. It tracks at most 100 000 keys per 60 s;
+ * are never added together. So when an outage ends inside a window, a
+ * key can get up to 2 × `maxPerMinute` in that window: up to the limit
+ * counted in memory during the outage, then up to the limit counted in
+ * Postgres after it. An upsert that commits but whose reply is lost is
+ * counted by both, which only denies sooner. It tracks at most 100 000
+ * keys per 60 s;
* a new key beyond that answers 503 as `'closed'` does. The counter
* lives in one process: control-api runs one replica today, so the
* in-memory limit holds for the whole service only while that is
* true. With N replicas a caller can get N × `maxPerMinute` requests
* per minute while Postgres is unavailable.
+ * - A 503 carries no `X-RateLimit-*` headers, including ones an earlier
+ * limiter on the same route set, because no count backs them.
*/
-export function rateLimitMiddleware(opts: {
- bucketType: string
- maxPerMinute: number
- getBucketKey: (req: Request) => string | null
- onBackendUnavailable: 'process-memory' | 'closed'
-}) {
+export function rateLimitMiddleware(
+ opts: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null }
+) {
+ const enforce = createRateLimitEnforcer(opts)
+ return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {
+ void (async () => {
+ try {
+ const key = opts.getBucketKey(req)
+ if (!key) {
+ // Fail-open: no key means we cannot attribute the call (usually a
+ // pre-auth path); let it through rather than 500.
+ next()
+ return
+ }
+ if (await enforce(req, res, key)) next()
+ } catch (err) {
+ next(err)
+ }
+ })()
+ }
+}
+
+/**
+ * The limiter behind `rateLimitMiddleware`, for a route that can only derive
+ * its key inside the handler (after authenticating the caller). Create it once
+ * per route set, not per request: with `'process-memory'` it owns the
+ * in-memory counter. Same semantics as `rateLimitMiddleware`.
+ */
+export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLimitEnforcer {
// While the backend is down every request on this bucket fails the same way,
// so the warn line is written once per minute; the counter records each one.
const unavailableLogThrottle = new LogThrottle(60_000)
@@ -71,6 +129,9 @@ export function rateLimitMiddleware(opts: {
'rate limit backend unavailable'
)
}
+ res.removeHeader('X-RateLimit-Limit')
+ res.removeHeader('X-RateLimit-Remaining')
+ res.removeHeader('X-RateLimit-Reset')
res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
res.setHeader('Cache-Control', 'no-store')
res.status(503).json({
@@ -109,58 +170,44 @@ export function rateLimitMiddleware(opts: {
res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })
}
- return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {
- void (async () => {
- try {
- const key = opts.getBucketKey(req)
- if (!key) {
- // Fail-open: no key means we cannot attribute the call (usually a
- // pre-auth path); let it through rather than 500.
- next()
- return
- }
-
- const result = await checkAndIncrement(key, opts.maxPerMinute)
- if (!result.backendAvailable) {
- if (processMemory === null) {
- answerUnavailable(req, res)
- return
- }
- const decision = await processMemory.hit(key)
- if (decision.outcome === 'key_cap_reached') {
- answerUnavailable(req, res)
- return
- }
- if (decision.outcome === 'denied') {
- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)
- answerDenied(req, res, key, {
- count: decision.totalHits,
- resetMs: decision.resetMs,
- source: 'process-memory',
- })
- return
- }
- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))
- res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))
- next()
- return
- }
- if (!result.allowed) {
- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)
- answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })
- return
- }
-
- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', String(result.remaining))
- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
- next()
- } catch (err) {
- next(err)
+ return async function enforce(req: Request, res: Response, rawKey: string): Promise<boolean> {
+ const key = boundedBucketKey(rawKey)
+ const result = await checkAndIncrement(key, opts.maxPerMinute)
+ if (!result.backendAvailable) {
+ if (processMemory === null) {
+ answerUnavailable(req, res)
+ return false
}
- })()
+ const decision = await processMemory.hit(key)
+ if (decision.outcome === 'key_cap_reached') {
+ answerUnavailable(req, res)
+ return false
+ }
+ if (decision.outcome === 'denied') {
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)
+ answerDenied(req, res, key, {
+ count: decision.totalHits,
+ resetMs: decision.resetMs,
+ source: 'process-memory',
+ })
+ return false
+ }
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)
+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
+ res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))
+ res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))
+ return true
+ }
+ if (!result.allowed) {
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)
+ answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })
+ return false
+ }
+
+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)
+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
+ res.setHeader('X-RateLimit-Remaining', String(result.remaining))
+ res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
+ return true
}
}
diff --git a/control-api/src/observability/logThrottle.ts b/control-api/src/observability/logThrottle.ts
index d497c19c5..b9168d489 100644
--- a/control-api/src/observability/logThrottle.ts
+++ b/control-api/src/observability/logThrottle.ts
@@ -1,3 +1,6 @@
+/** Distinct keys one throttle tracks before new keys share a single entry. */
+export const LOG_THROTTLE_MAX_KEYS = 1_000
+
/**
* Bounds how many lines one event writes: at most one per key per interval.
* A suppressed event is only counted, and the next admitted line for that key
@@ -9,6 +12,12 @@
* interval, so the map is bounded by the keys active in the last two intervals.
* A count still pending when its key is swept is lost from the log, not from
* the metrics.
+ *
+ * Keys often come from clients, and a client can send a new one on every
+ * request. Once `maxKeys` keys are tracked, every key not already tracked shares
+ * one overflow entry: together they get at most one line per interval, and that
+ * line carries their combined suppressed count. So one interval writes at most
+ * `maxKeys + 1` lines, and the map holds at most `maxKeys + 1` entries.
*/
export class LogThrottle {
private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>()
@@ -16,9 +25,13 @@ export class LogThrottle {
constructor(
private readonly intervalMs: number,
- private readonly now: () => number = () => Date.now()
+ private readonly now: () => number = () => Date.now(),
+ private readonly maxKeys: number = LOG_THROTTLE_MAX_KEYS
) {
if (!(intervalMs > 0)) throw new Error('LogThrottle: intervalMs must be > 0')
+ if (!Number.isSafeInteger(maxKeys) || maxKeys < 1) {
+ throw new Error('LogThrottle: maxKeys must be a positive integer')
+ }
this.lastSweep = now()
}
@@ -35,12 +48,14 @@ export class LogThrottle {
admit(key: string): number | undefined {
const now = this.now()
this.sweep(now)
- const entry = this.entries.get(key)
+ const trackedKey =
+ this.entries.has(key) || this.entries.size < this.maxKeys ? key : OVERFLOW_KEY
+ const entry = this.entries.get(trackedKey)
if (entry && now - entry.emittedAt < this.intervalMs) {
entry.suppressed += 1
return undefined
}
- this.entries.set(key, { emittedAt: now, suppressed: 0 })
+ this.entries.set(trackedKey, { emittedAt: now, suppressed: 0 })
return entry?.suppressed ?? 0
}
@@ -52,3 +67,7 @@ export class LogThrottle {
this.lastSweep = now
}
}
+
+// A NUL byte cannot appear in a key built from a header, a URL or a hex hash,
+// so the overflow entry cannot collide with a real key.
+const OVERFLOW_KEY = '\u0000overflow'
diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.ts
index c40efc41b..a02f7047f 100644
--- a/control-api/src/routes/admin/auth.ts
+++ b/control-api/src/routes/admin/auth.ts
@@ -1,6 +1,7 @@
import { type Request, type RequestHandler, Router } from 'express'
import bcrypt from 'bcryptjs'
import { ipKeyGenerator } from 'express-rate-limit'
+import { createHash } from 'node:crypto'
import { config } from '../../config.js'
import { pool } from '../../db.js'
import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'
@@ -79,7 +80,11 @@ function publicAdminTokenRateLimits(routeName: string): [RequestHandler, Request
.trim()
.toLowerCase()
const login = String(req.body?.username || req.body?.login || '').trim()
- return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${email || login || token.slice(0, 48) || 'missing'}`
+ // The submitted value has no length bound before this point, and the key
+ // is stored per window, so it enters the key as a fixed-size digest. The
+ // digest also keeps emails out of the bucket table.
+ const submitted = email || login || token.slice(0, 48) || 'missing'
+ return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${createHash('sha256').update(submitted).digest('hex')}`
},
onBackendUnavailable: 'process-memory',
})
diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.ts
index e8dfb0aa8..48dd8690d 100644
--- a/control-api/src/routes/external/auth.ts
+++ b/control-api/src/routes/external/auth.ts
@@ -1,4 +1,5 @@
import { Router } from 'express'
+import { createHash } from 'node:crypto'
import { pool } from '../../db.js'
import type { K8sGateway } from '../../k8s.js'
import { isCurrentExternalSession } from '../../middleware/externalSessionAuth.js'
@@ -127,7 +128,12 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {
const email = String(req.body?.email || '')
.trim()
.toLowerCase()
- return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`
+ // The email is client-supplied and unbounded before this limiter, so it
+ // enters the key as a fixed-size digest: an oversized value cannot grow
+ // the limiter's storage, and the same email still maps to one bucket.
+ return email
+ ? `profile_password_reset:${createHash('sha256').update(email).digest('hex')}`
+ : `profile_password_reset_ip:${req.ip}`
},
onBackendUnavailable: 'process-memory',
}),
diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
index 948d05d55..288e345d5 100644
--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts
@@ -3,15 +3,16 @@ import { config } from '../../config.js'
import type { K8sGateway } from '../../k8s.js'
import { mcpHostHttpMetrics } from '../../middleware/mcpHostHttpMetrics.js'
import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'
-import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'
-import { rateLimitHitsTotal } from '../../observability/metrics.js'
+import {
+ createRateLimitEnforcer,
+ rateLimitMiddleware,
+} from '../../middleware/rateLimitMiddleware.js'
import {
acknowledgeNotificationDelivery,
claimNotificationDeliveries,
failNotificationDelivery,
resolvePendingWorkflowApprovalDelivery,
} from '../../services/notificationDeliveryQueueService.js'
-import { checkAndIncrement } from '../../services/rateLimiterService.js'
import type { DirectRunAttributionBindingService } from '../../services/tracing/directRunAttributionBindingService.js'
import {
InvalidWorkflowApprovalRunBindingError,
@@ -436,47 +437,23 @@ function classifyMcpHostCaller(
: 'first-party-mcp-host'
}
-async function enforceMediumResolveRateLimit(
- req: Request,
- res: Response,
- callerKey: string
-): Promise<boolean> {
- const bucketType = 'mcp_host_workflow_approval_medium_resolve'
- const bucketKey = `medium-resolve:${callerKey}`
- const result = await checkAndIncrement(bucketKey, config.approvalRlRequestPerMin)
- if (!result.allowed) {
- rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'denied' }, 1)
- const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))
- res.setHeader('Retry-After', String(retryAfterSec))
- res.setHeader('X-RateLimit-Limit', String(config.approvalRlRequestPerMin))
- res.setHeader('X-RateLimit-Remaining', '0')
- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
- req.log?.warn(
- {
- event: 'rate_limit_denied',
- bucketType,
- bucketKey,
- count: result.count,
- maxPerMinute: config.approvalRlRequestPerMin,
- },
- 'rate limit exceeded'
- )
- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })
- return false
- }
-
- rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'allowed' }, 1)
- res.setHeader('X-RateLimit-Limit', String(config.approvalRlRequestPerMin))
- res.setHeader('X-RateLimit-Remaining', String(result.remaining))
- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
- return true
-}
-
export function createUserApprovalRequestsRoutes(
gateway: K8sGateway,
directRunAttributionBindingService: DirectRunAttributionBindingService
): Router {
const router = Router()
+ // The caller key is known only after the handler authenticates the caller,
+ // so these routes use the enforcer instead of the middleware. One instance
+ // serves all four routes: they share the `medium-resolve:<caller>` bucket.
+ // A 503 would fail the approval a user is waiting on, so while Postgres
+ // cannot count, the limit is counted in this process's memory.
+ const enforceMediumResolveRateLimit = createRateLimitEnforcer({
+ bucketType: 'mcp_host_workflow_approval_medium_resolve',
+ maxPerMinute: config.approvalRlRequestPerMin,
+ onBackendUnavailable: 'process-memory',
+ })
+ const enforceMediumResolve = (req: Request, res: Response, callerKey: string) =>
+ enforceMediumResolveRateLimit(req, res, `medium-resolve:${callerKey}`)
router.post(
'/workflow-approvals/request',
@@ -866,7 +843,7 @@ export function createUserApprovalRequestsRoutes(
if (!caller) return
if (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) return
const callerKey = getMcpHostCallerKey(caller.claims)
- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return
+ if (!(await enforceMediumResolve(req, res, callerKey))) return
const parsedIdentity = parseProviderIdentityFromBody(req.body)
if (!parsedIdentity.ok) {
@@ -989,7 +966,7 @@ export function createUserApprovalRequestsRoutes(
if (!caller) return
if (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) return
const callerKey = getMcpHostCallerKey(caller.claims)
- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return
+ if (!(await enforceMediumResolve(req, res, callerKey))) return
const deliveries = await claimNotificationDeliveries({
medium: req.query.medium,
@@ -1287,7 +1264,7 @@ export function createUserApprovalRequestsRoutes(
if (!caller) return
if (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) return
const callerKey = getMcpHostCallerKey(caller.claims)
- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return
+ if (!(await enforceMediumResolve(req, res, callerKey))) return
const body = requestBodyRecord(req.body)
const code = String(body.code || '').trim()
@@ -1337,7 +1314,7 @@ export function createUserApprovalRequestsRoutes(
if (!caller) return
if (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) return
const callerKey = getMcpHostCallerKey(caller.claims)
- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return
+ if (!(await enforceMediumResolve(req, res, callerKey))) return
const parsedIdentity = parseProviderIdentityFromBody(req.body)
if (!parsedIdentity.ok) {
diff --git a/control-api/src/services/rateLimitBucketKey.ts b/control-api/src/services/rateLimitBucketKey.ts
new file mode 100644
index 000000000..847a0daf5
--- /dev/null
+++ b/control-api/src/services/rateLimitBucketKey.ts
@@ -0,0 +1,24 @@
+import { createHash } from 'node:crypto'
+
+/** Longest bucket key, in UTF-8 bytes, that the rate limiters store as given. */
+export const MAX_RATE_LIMIT_BUCKET_KEY_BYTES = 512
+
+const LONG_KEY_PREFIX = 'sha256-long-key:'
+
+/**
+ * The form of a bucket key that both limiter backends store.
+ *
+ * Several key builders include a value the client submits (an email, a login,
+ * a token prefix, a route parameter). Without a bound, an unauthenticated
+ * client can send one oversized value per request: Postgres rejects the upsert
+ * (the btree entry limit is 2704 bytes), so the request falls through to the
+ * process-memory counter, which keeps each key for its whole window. A key over
+ * `MAX_RATE_LIMIT_BUCKET_KEY_BYTES` is therefore replaced by its SHA-256. The
+ * same input always gives the same stored key, so an oversized value is still
+ * counted and limited like any other; only its size changes. Keys at or under
+ * the bound are returned unchanged, so existing buckets keep their keys.
+ */
+export function boundedBucketKey(key: string): string {
+ if (Buffer.byteLength(key, 'utf8') <= MAX_RATE_LIMIT_BUCKET_KEY_BYTES) return key
+ return `${LONG_KEY_PREFIX}${createHash('sha256').update(key).digest('hex')}`
+}
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index ba2adc912..5ee268d78 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -4,6 +4,7 @@ import { pool, rateLimitPool } from '../db.js'
import { LogThrottle } from '../observability/logThrottle.js'
import { rootLogger } from '../observability/logger.js'
import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'
+import { boundedBucketKey } from './rateLimitBucketKey.js'
/**
* PG-backed sliding-window-ish rate limiter.
@@ -22,7 +23,8 @@ import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'
const WINDOW_MS = 60_000
-// One rate_limit_db_error line per bucket per window.
+// One rate_limit_db_error line per bucket key per window. Keys come from
+// clients, so the throttle also caps how many distinct keys it tracks.
const dbErrorLogThrottle = new LogThrottle(WINDOW_MS)
export type RateLimitCheck = {
@@ -87,6 +89,9 @@ export async function checkAndIncrementWithQuery(
if (!Number.isSafeInteger(cost) || cost < 1) throw new Error('rate limit cost must be positive')
const windowStartMs = currentWindowStartMs(nowMs)
const resetMs = windowStartMs + WINDOW_MS
+ // Callers that build keys from client input without calling the middleware
+ // get the same bound; an already bounded key is returned unchanged.
+ const storedKey = boundedBucketKey(bucketKey)
let result: { rows: unknown[] } | null | undefined
try {
@@ -96,7 +101,7 @@ export async function checkAndIncrementWithQuery(
ON CONFLICT (bucket_key, window_start_ms) DO UPDATE
SET count = rate_limit_buckets.count + EXCLUDED.count
RETURNING count`,
- [bucketKey, windowStartMs, cost]
+ [storedKey, windowStartMs, cost]
)
} catch (err) {
// The service does not decide admission. It reports backendAvailable:false
@@ -107,10 +112,10 @@ export async function checkAndIncrementWithQuery(
// the external GFS limiter logs as hashedKey, so the lines still join. It
// is a correlation id, not anonymization: unsalted, so a low-entropy key
// (an IP, an email) can be recovered by guessing. While the backend is
- // down every request lands here, so the line is throttled per bucket and
- // the counter keeps the full count.
+ // down every request lands here, so the line is throttled per bucket key
+ // and the counter keeps the full count.
rateLimitBackendErrorsTotal.inc()
- const hashedKey = createHash('sha256').update(bucketKey).digest('hex')
+ const hashedKey = createHash('sha256').update(storedKey).digest('hex')
const suppressed = dbErrorLogThrottle.admit(hashedKey)
if (suppressed !== undefined) {
rootLogger.warn(
diff --git a/control-api/test/app.memberRegistrationUnavailable.test.ts b/control-api/test/app.memberRegistrationUnavailable.test.ts
index 099ef9dd0..b163c8c1e 100644
--- a/control-api/test/app.memberRegistrationUnavailable.test.ts
+++ b/control-api/test/app.memberRegistrationUnavailable.test.ts
@@ -18,6 +18,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({
rateLimitMiddleware:
() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>
next(),
+ createRateLimitEnforcer: () => async () => true,
}))
describe('app: member-registration unavailability maps to 503 (real error middleware)', () => {
diff --git a/control-api/test/app.publishImageRefIdentity.test.ts b/control-api/test/app.publishImageRefIdentity.test.ts
index 1b6bb8f6b..bec070aed 100644
--- a/control-api/test/app.publishImageRefIdentity.test.ts
+++ b/control-api/test/app.publishImageRefIdentity.test.ts
@@ -38,6 +38,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({
rateLimitMiddleware:
() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>
next(),
+ createRateLimitEnforcer: () => async () => true,
}))
const baseBody = (imageRef: string) => ({
diff --git a/control-api/test/app.publisherUiEnabled.test.ts b/control-api/test/app.publisherUiEnabled.test.ts
index 7d2f68854..24a57eaca 100644
--- a/control-api/test/app.publisherUiEnabled.test.ts
+++ b/control-api/test/app.publisherUiEnabled.test.ts
@@ -40,6 +40,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({
rateLimitMiddleware:
() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>
next(),
+ createRateLimitEnforcer: () => async () => true,
}))
describe('GET /admin/registry/publish-scope: publisherUiEnabled (effective, through the route)', () => {
diff --git a/control-api/test/app.registryCatalogUnavailable.test.ts b/control-api/test/app.registryCatalogUnavailable.test.ts
index 3cd09a987..5108e7b83 100644
--- a/control-api/test/app.registryCatalogUnavailable.test.ts
+++ b/control-api/test/app.registryCatalogUnavailable.test.ts
@@ -39,6 +39,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({
rateLimitMiddleware:
() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>
next(),
+ createRateLimitEnforcer: () => async () => true,
}))
describe('app: registry-unavailable forwarding at the global error handler (real middleware)', () => {
diff --git a/control-api/test/db.poolFactory.test.ts b/control-api/test/db.poolFactory.test.ts
index d2ca8309b..893287ead 100644
--- a/control-api/test/db.poolFactory.test.ts
+++ b/control-api/test/db.poolFactory.test.ts
@@ -109,6 +109,26 @@ describe('rate limiter Postgres pool', () => {
})
})
+ it.each([
+ ['CORE_POOL_MAX', ' ', 'max', 10, 0],
+ ['CORE_POOL_MAX', '', 'max', 10, 0],
+ ['RATE_LIMIT_POOL_MAX', '\t', 'max', 6, 1],
+ ['RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS', ' ', 'statement_timeout', 3_000, 1],
+ ] as const)(
+ 'treats a blank %s (%j) as unset, as config.ts does, and uses the default',
+ async (name, value, field, expected, poolIndex) => {
+ process.env[name] = value
+ try {
+ await import('../src/db.js')
+ // Witness: module load built both pools; the one reading the value got the default.
+ expect(fakePools).toHaveLength(2)
+ expect(fakePools[poolIndex]?.config[field]).toBe(expected)
+ } finally {
+ delete process.env[name]
+ }
+ }
+ )
+
it.each([
// [env, value, range, pools built before the refusal (core is built first)]
['RATE_LIMIT_POOL_MAX', '17', '[1, 16]', 1],
diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.ts
index adc8669fd..821fb7f5b 100644
--- a/control-api/test/gfsUploadAdmission.test.ts
+++ b/control-api/test/gfsUploadAdmission.test.ts
@@ -280,8 +280,12 @@ describe('GFS upload admission', () => {
checkMock.mockResolvedValue(rateCheck({ backendAvailable: false }))
const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')
expect(response.status).toBe(503)
- expect(response.body).toEqual({ error: 'gfs_upload_admission_unavailable' })
+ expect(response.body).toEqual({
+ error: 'gfs_upload_admission_unavailable',
+ retryAfterSeconds: 2,
+ })
expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['cache-control']).toBe('no-store')
expect(checkMock).toHaveBeenCalled()
expect(acquireMock).not.toHaveBeenCalled()
})
diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.ts
index 9406e3cbf..fcd2728b4 100644
--- a/control-api/test/middleware.externalGfsRateLimit.test.ts
+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts
@@ -304,6 +304,11 @@ describe('external GFS rate boundary', () => {
expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })
expect(response.headers['retry-after']).toBe('2')
expect(response.headers['cache-control']).toBe('no-store')
+ // The three buckets that admitted the request wrote X-RateLimit-*; the 503
+ // does not carry their counts, which describe a budget it did not charge.
+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()
+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()
+ expect(response.headers['x-ratelimit-reset']).toBeUndefined()
expect(checkAndIncrement).toHaveBeenCalledTimes(4)
expect(checkAndIncrement).toHaveBeenLastCalledWith(
`gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,
diff --git a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
index 0138cbcb9..95749bfa2 100644
--- a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
+++ b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
@@ -120,6 +120,11 @@ describe('external session lifecycle gate', () => {
],
// Five characters, like a SQLSTATE, but a socket error: no SQLSTATE class.
['a broken pipe', Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })],
+ // Node raises this when every address of a multi-address host refuses.
+ [
+ 'a refused socket on every address',
+ Object.assign(new AggregateError([], 'connect ECONNREFUSED'), { code: 'ECONNREFUSED' }),
+ ],
['SQLSTATE 08006 connection_failure', databaseError('08006')],
['SQLSTATE 53300 too_many_connections', databaseError('53300')],
['SQLSTATE 57P01 admin_shutdown', databaseError('57P01')],
@@ -138,6 +143,15 @@ describe('external session lifecycle gate', () => {
['SQLSTATE 22P02 invalid_text_representation', databaseError('22P02')],
['SQLSTATE 42P01 undefined_table', databaseError('42P01')],
['SQLSTATE 42703 undefined_column', databaseError('42703')],
+ // Defects in this process's code, not in the connection to PostgreSQL.
+ ['a TypeError', new TypeError("Cannot read properties of undefined (reading 'query')")],
+ ['a RangeError', new RangeError('Invalid array length')],
+ [
+ 'a Node argument error',
+ Object.assign(new TypeError('The "string" argument must be of type string'), {
+ code: 'ERR_INVALID_ARG_TYPE',
+ }),
+ ],
])('passes %s to the error handler as a 500, not a 503', async (_label, error) => {
verifyToken.mockReturnValueOnce(claims)
poolQuery.mockRejectedValueOnce(error)
diff --git a/control-api/test/middleware.processMemoryRateLimiter.test.ts b/control-api/test/middleware.processMemoryRateLimiter.test.ts
index 0cd45b10c..a025a154c 100644
--- a/control-api/test/middleware.processMemoryRateLimiter.test.ts
+++ b/control-api/test/middleware.processMemoryRateLimiter.test.ts
@@ -46,6 +46,32 @@ describe('ProcessMemoryRateLimiter', () => {
expect(await limiter.hit('c')).toMatchObject({ outcome: 'allowed', totalHits: 1 })
})
+ it('keeps counting a key whose window is still running after the tracked-key set is cleared', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(START_MS)
+ const limiter = new ProcessMemoryRateLimiter(5, 2)
+
+ // 'b' opens its store window at +30 s, so it runs until +90 s.
+ vi.setSystemTime(START_MS + 30_000)
+ expect(await limiter.hit('b')).toMatchObject({ outcome: 'allowed', totalHits: 1 })
+
+ // At +60 s the class's own window rolls over and its key set is cleared;
+ // two new keys fill it to the cap.
+ vi.setSystemTime(START_MS + 60_000)
+ expect((await limiter.hit('x')).outcome).toBe('allowed')
+ expect((await limiter.hit('y')).outcome).toBe('allowed')
+
+ vi.setSystemTime(START_MS + 61_000)
+ // Witness: the cap is in force for a key with no running window.
+ expect(await limiter.hit('z')).toEqual({ outcome: 'key_cap_reached' })
+ // 'b' is still inside its window, so it is counted, not refused.
+ expect(await limiter.hit('b')).toEqual({
+ outcome: 'allowed',
+ totalHits: 2,
+ resetMs: START_MS + 90_000,
+ })
+ })
+
it('tracks at most 100 000 keys by default', async () => {
expect(PROCESS_MEMORY_MAX_TRACKED_KEYS).toBe(100_000)
const limiter = new ProcessMemoryRateLimiter(1)
diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.ts
index 03f1b68e6..d096a78de 100644
--- a/control-api/test/middleware.rateLimitMiddleware.test.ts
+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts
@@ -42,18 +42,6 @@ function unavailable(): RateLimitCheck {
}
}
-// Compile-time check only. control-api's tsconfig covers src/, so vitest does
-// not typecheck this file; run tsc over it to verify the expectation below.
-export function buildWithRemovedOpenMode() {
- return rateLimitMiddleware({
- bucketType: 'unit_bucket',
- maxPerMinute: LIMIT,
- getBucketKey: () => 'unit:key',
- // @ts-expect-error 'open' is no longer a mode
- onBackendUnavailable: 'open',
- })
-}
-
function appWith(onBackendUnavailable: 'process-memory' | 'closed') {
const handler = vi.fn((_req: express.Request, res: express.Response) => {
res.status(204).end()
@@ -232,6 +220,9 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
setHeader(name: string, value: string) {
headers[name.toLowerCase()] = value
},
+ removeHeader(name: string) {
+ delete headers[name.toLowerCase()]
+ },
status(code: number) {
this.statusCode = code
return this
@@ -275,6 +266,88 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
])
}, 60_000)
+ it("'closed' 503 carries no X-RateLimit-* headers left by an earlier limiter on the route", async () => {
+ // First limiter counts and admits; the second cannot reach its backend.
+ checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(unavailable())
+ const handler = vi.fn((_req: express.Request, res: express.Response) => res.status(204).end())
+ const app = express()
+ app.get(
+ '/limited',
+ (req, _res, next) => {
+ ;(req as unknown as { log: { warn: () => void } }).log = { warn: () => {} }
+ next()
+ },
+ rateLimitMiddleware({
+ bucketType: 'first_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'first:key',
+ onBackendUnavailable: 'closed',
+ }),
+ rateLimitMiddleware({
+ bucketType: 'second_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'second:key',
+ onBackendUnavailable: 'closed',
+ }),
+ handler
+ )
+
+ const response = await request(app).get('/limited')
+
+ expect(response.status).toBe(503)
+ // Witness: both limiters ran, the first one admitting the request.
+ expect(checkAndIncrement.mock.calls).toEqual([
+ ['first:key', LIMIT],
+ ['second:key', LIMIT],
+ ])
+ expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()
+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()
+ expect(response.headers['x-ratelimit-reset']).toBeUndefined()
+ expect(handler).not.toHaveBeenCalled()
+ })
+
+ it('bounds an oversized bucket key before it reaches either backend, and keeps it stable', async () => {
+ const huge = `unit:${'x'.repeat(10_000)}`
+ checkAndIncrement.mockResolvedValue(unavailable())
+ const middleware = rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => huge,
+ onBackendUnavailable: 'process-memory',
+ })
+ const increment = vi.spyOn(MemoryStore.prototype, 'increment')
+ try {
+ const app = express()
+ app.get(
+ '/limited',
+ (req, _res, next) => {
+ ;(req as unknown as { log: { warn: () => void } }).log = { warn: () => {} }
+ next()
+ },
+ middleware,
+ (_req, res) => res.status(204).end()
+ )
+
+ const first = await request(app).get('/limited')
+ const second = await request(app).get('/limited')
+
+ expect([first.status, second.status]).toEqual([204, 204])
+ // Same key both times, so the in-memory count still accumulates.
+ expect(second.headers['x-ratelimit-remaining']).toBe(String(LIMIT - 2))
+ // Witness: the Postgres limiter and the in-memory store were both reached.
+ expect(checkAndIncrement).toHaveBeenCalledTimes(2)
+ expect(increment).toHaveBeenCalledTimes(2)
+ const postgresKeys = checkAndIncrement.mock.calls.map(([key]) => key as string)
+ const memoryKeys = increment.mock.calls.map(([key]) => key)
+ const bounded = `sha256-long-key:${createHash('sha256').update(huge).digest('hex')}`
+ expect(postgresKeys).toEqual([bounded, bounded])
+ expect(memoryKeys).toEqual([bounded, bounded])
+ } finally {
+ increment.mockRestore()
+ }
+ })
+
it("'closed' still admits a counted request and still answers 429 over the limit", async () => {
checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))
const { app, handler } = appWith('closed')
diff --git a/control-api/test/observability.logThrottle.test.ts b/control-api/test/observability.logThrottle.test.ts
index ab8054cf6..28dbeffcf 100644
--- a/control-api/test/observability.logThrottle.test.ts
+++ b/control-api/test/observability.logThrottle.test.ts
@@ -52,7 +52,39 @@ describe('LogThrottle', () => {
expect(throttle.trackedKeyCount).toBe(1)
})
- it('rejects a non-positive interval', () => {
+ it('sends every key past maxKeys to one shared overflow entry, so an interval writes at most maxKeys + 1 lines', () => {
+ let t = 1_000
+ const throttle = new LogThrottle(60_000, () => t, 3)
+
+ const tracked = ['a', 'b', 'c'].map(key => throttle.admit(key))
+ const overflow = Array.from({ length: 50 }, (_, i) => throttle.admit(`new-${i}`))
+
+ // Witness: the three tracked keys each got their line.
+ expect(tracked).toEqual([0, 0, 0])
+ // Fifty distinct new keys share one line between them.
+ expect(overflow.filter(r => r !== undefined)).toEqual([0])
+ expect(throttle.trackedKeyCount).toBe(4)
+ // A tracked key is still throttled on its own entry.
+ expect(throttle.admit('a')).toBeUndefined()
+
+ t += 60_000
+ // The overflow line carries the combined count of the 49 it suppressed.
+ expect(throttle.admit('new-999')).toBe(49)
+ })
+
+ it('tracks at most 1 000 keys by default', () => {
+ const t = 1_000
+ const throttle = new LogThrottle(60_000, () => t)
+ const admitted = Array.from({ length: 5_000 }, (_, i) => throttle.admit(`k${i}`)).filter(
+ r => r !== undefined
+ )
+ expect(admitted).toHaveLength(1_001)
+ expect(throttle.trackedKeyCount).toBe(1_001)
+ })
+
+ it('rejects a non-positive interval and a non-positive key cap', () => {
expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/)
+ expect(() => new LogThrottle(1_000, () => 0, 0)).toThrow(/maxKeys must be a positive integer/)
+ expect(() => new LogThrottle(1_000, () => 0, 1.5)).toThrow(/maxKeys must be a positive integer/)
})
})
diff --git a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts
index f22ec20f8..b5efad0b6 100644
--- a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts
+++ b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts
@@ -1,8 +1,11 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
+import { createHash } from 'node:crypto'
import request from 'supertest'
import { config } from '../src/config.js'
+const sha256 = (value: string) => createHash('sha256').update(value).digest('hex')
+
/**
* The public control-admin token routes (password reset, invitation, email
* confirmation) are called before login. Each route has two buckets:
@@ -212,11 +215,37 @@ describe('public control-admin token limiter', () => {
expect(limiterCalls).toEqual([
{
- key: `control-admin-public:${routeName}:203.0.113.7:[email protected]`,
+ key: `control-admin-public:${routeName}:203.0.113.7:${sha256('[email protected]')}`,
max: PER_VALUE,
},
{ key: `control-admin-public-ip:${routeName}:203.0.113.7`, max: PER_IP },
])
}
)
+
+ it('keys the per-value bucket on a fixed-size digest of the submitted value, whatever its size', async () => {
+ const { app } = buildApp()
+ // Under express.json's default 100 kB limit here; production allows more.
+ const huge = `${'a'.repeat(50_000)}@corp.test`
+ await request(app)
+ .post('/admin/auth/password-reset/request')
+ .set('x-forwarded-for', '203.0.113.9')
+ .send({ email: huge })
+ await request(app)
+ .post('/admin/auth/password-reset/request')
+ .set('x-forwarded-for', '203.0.113.9')
+ .send({ email: huge })
+
+ // Liveness witness: both requests reached the per-value limiter.
+ const perValueKeys = limiterCalls
+ .map(call => call.key)
+ .filter(key => key.startsWith('control-admin-public:'))
+ expect(perValueKeys).toHaveLength(2)
+ // Same value, same bucket; the key does not grow with the value.
+ expect(perValueKeys[0]).toBe(perValueKeys[1])
+ expect(perValueKeys[0]).toBe(
+ `control-admin-public:password_reset_request:203.0.113.9:${sha256(huge)}`
+ )
+ expect(Buffer.byteLength(perValueKeys[0] as string)).toBeLessThan(200)
+ })
})
diff --git a/control-api/test/routes.externalAuth.passwordResetLimiter.test.ts b/control-api/test/routes.externalAuth.passwordResetLimiter.test.ts
new file mode 100644
index 000000000..ef95e3090
--- /dev/null
+++ b/control-api/test/routes.externalAuth.passwordResetLimiter.test.ts
@@ -0,0 +1,91 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import { createHash } from 'node:crypto'
+import request from 'supertest'
+
+/**
+ * `POST /external/auth/password-reset/request` is called before login and keys
+ * its limiter on the submitted email. The email has no length bound before the
+ * limiter, so it must enter the bucket key as a fixed-size digest: otherwise a
+ * client can grow the limiter's storage with one oversized value per request.
+ */
+
+const limiterCalls = vi.hoisted(() => [] as Array<{ key: string; max: number }>)
+
+vi.mock('../src/services/rateLimiterService.js', () => ({
+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,
+ checkAndIncrement: vi.fn(async (key: string, max: number) => {
+ limiterCalls.push({ key, max })
+ return {
+ allowed: true,
+ remaining: max - 1,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count: 1,
+ backendAvailable: true,
+ }
+ }),
+}))
+
+const directory = vi.hoisted(() => ({
+ getTeamAgents: vi.fn(),
+ getUserAgents: vi.fn(),
+ googleLoginData: vi.fn(),
+ passwordLoginData: vi.fn(),
+ requestProfilePasswordReset: vi.fn(async () => undefined),
+}))
+vi.mock('../src/services/directory/index.js', () => directory)
+vi.mock('../src/db.js', () => ({ pool: { query: vi.fn() } }))
+
+const { createExternalAuthRouter } = await import('../src/routes/external/auth.js')
+
+const sha256 = (value: string) => createHash('sha256').update(value).digest('hex')
+
+function buildApp() {
+ const app = express()
+ app.set('trust proxy', 1)
+ app.use(express.json())
+ app.use(createExternalAuthRouter({} as never))
+ return app
+}
+
+describe('external password reset request limiter key', () => {
+ beforeEach(() => {
+ limiterCalls.length = 0
+ directory.requestProfilePasswordReset.mockClear()
+ })
+
+ it('keys the bucket on a digest of the normalized email', async () => {
+ const response = await request(buildApp())
+ .post('/external/auth/password-reset/request')
+ .send({ email: ' [email protected] ' })
+
+ expect(response.status).toBe(200)
+ // Liveness witness: the handler ran after the limiter.
+ expect(directory.requestProfilePasswordReset).toHaveBeenCalledWith('[email protected]')
+ expect(limiterCalls).toEqual([
+ { key: `profile_password_reset:${sha256('[email protected]')}`, max: 5 },
+ ])
+ })
+
+ it('gives an oversized email the same fixed-size key on every request', async () => {
+ const app = buildApp()
+ // Under express.json's default 100 kB limit here; production allows more.
+ const huge = `${'a'.repeat(50_000)}@example.test`
+ await request(app).post('/external/auth/password-reset/request').send({ email: huge })
+ await request(app).post('/external/auth/password-reset/request').send({ email: huge })
+
+ expect(limiterCalls).toHaveLength(2)
+ expect(limiterCalls[0]?.key).toBe(`profile_password_reset:${sha256(huge)}`)
+ expect(limiterCalls[1]?.key).toBe(limiterCalls[0]?.key)
+ })
+
+ it('keys a request with no email on the source address', async () => {
+ await request(buildApp())
+ .post('/external/auth/password-reset/request')
+ .set('x-forwarded-for', '198.51.100.4')
+ .send({})
+
+ expect(limiterCalls).toEqual([{ key: 'profile_password_reset_ip:198.51.100.4', max: 5 }])
+ })
+})
diff --git a/control-api/test/routes.mcp-host.user-approval-requests.test.ts b/control-api/test/routes.mcp-host.user-approval-requests.test.ts
index b76350597..192aeb533 100644
--- a/control-api/test/routes.mcp-host.user-approval-requests.test.ts
+++ b/control-api/test/routes.mcp-host.user-approval-requests.test.ts
@@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import request from 'supertest'
import { createApp } from '../src/app.js'
+import { config } from '../src/config.js'
import * as notificationDeliveryQueueService from '../src/services/notificationDeliveryQueueService.js'
import * as userApprovalRequestService from '../src/services/userApprovalRequestService.js'
import * as workflowApprovalMediumLinkSessionService from '../src/services/workflowApprovalMediumLinkSessionService.js'
@@ -1269,6 +1270,38 @@ describe('User Approval Request Routes', () => {
).not.toHaveBeenCalled()
})
+ it('keeps counting in process memory when the Postgres limiter cannot count, and answers 429 over the limit', async () => {
+ // The db mock has no rateLimitPool, so every checkAndIncrement reports
+ // backendAvailable:false, as it does when the limiter pool is saturated.
+ const previousLimit = config.approvalRlRequestPerMin
+ config.approvalRlRequestPerMin = 3
+ try {
+ const limitedApp = createApp(gateway as never)
+ const token = issueProviderDecisionControlToken(['workflow:approval:resolve'])
+ const send = () =>
+ request(limitedApp)
+ .post('/api/v1/workflow-approval-mediums/resolve')
+ .set('Authorization', `Bearer ${token}`)
+ .send({ providerIdentity: { medium: 'telegram', providerUserId: '123456' } })
+
+ const admitted = [await send(), await send(), await send()]
+ const refused = await send()
+
+ // Witness: the first three reached the handler's validation, each one
+ // counted by the in-memory limiter.
+ expect(admitted.map(r => r.status)).toEqual([400, 400, 400])
+ expect(admitted.map(r => r.body.error)).toEqual(
+ Array(3).fill('provider channel identity is required')
+ )
+ expect(admitted.map(r => r.headers['x-ratelimit-remaining'])).toEqual(['2', '1', '0'])
+ expect(refused.status).toBe(429)
+ expect(refused.body).toMatchObject({ error: 'Too Many Requests' })
+ expect(refused.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)
+ } finally {
+ config.approvalRlRequestPerMin = previousLimit
+ }
+ })
+
it('requires stable provider channel identity', async () => {
const token = issueProviderDecisionControlToken(['workflow:approval:resolve'])
diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.ts
index 6a3928140..522a9c6d4 100644
--- a/control-api/test/services.rateLimiter.test.ts
+++ b/control-api/test/services.rateLimiter.test.ts
@@ -208,7 +208,7 @@ describe('rateLimiterService', () => {
await replacement.release()
})
- it('fails open when the limiter pool query throws (DB error)', async () => {
+ it('reports the backend unavailable when the limiter pool query throws (DB error); the caller decides the policy', async () => {
mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
const r = await checkAndIncrement('test:bucket:failopen', 5)
expect(r.allowed).toBe(true)
@@ -271,7 +271,7 @@ describe('rateLimiterService', () => {
}
})
- it('fails open when the limiter pool query returns empty rows', async () => {
+ it('reports the backend unavailable when the limiter pool query returns empty rows', async () => {
mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })
const r = await checkAndIncrement('test:bucket:emptyrows', 5)
expect(r.allowed).toBe(true)