Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): close the remaining limiter gaps from the second review round
Alfredo Lopez7dc6e80fab66
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex f1b260b97..e82d0cdbe 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.618",+ "version": "0.1.619","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.618",+ "version": "0.1.619","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 2bee1119c..39a202743 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.618",+ "version": "0.1.619","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 9a6e8de87..08b9a5abb 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -136,14 +136,14 @@ const DEFAULT_RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS = 3_000export const RATE_LIMIT_POOL_SESSION_OPTIONS = '-c synchronous_commit=off'/**- * Unset (or empty) means the default. A set value outside `[min, max]`, or one- * that is not canonical decimal integer text, stops startup instead of- * silently becoming the default. Number() alone would also read '6.0', '0x6',- * '6e0' and ' 6' as 6.+ * Unset, empty or whitespace-only means the default, as in config.ts. A set+ * value outside `[min, max]`, or one that is not canonical decimal integer+ * text, stops startup instead of silently becoming the default. Number() alone+ * would also read '6.0', '0x6', '6e0' and ' 6' as 6.*/function boundedEnvInteger(name: string, fallback: number, min: number, max: number): number {const raw = process.env[name]- if (!raw) return fallback+ if (raw === undefined || raw.trim() === '') return fallbackconst value = Number(raw)if (!/^(0|[1-9]\d*)$/.test(raw) || value < min || value > max) {throw new Error(`${name} must be an integer in [${min}, ${max}], got ${JSON.stringify(raw)}`)diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.tsindex b836a1575..88510b723 100644--- a/control-api/src/middleware/externalGfsRateLimit.ts+++ b/control-api/src/middleware/externalGfsRateLimit.ts@@ -379,6 +379,11 @@ async function enforceBuckets(input: {latencyMs,authorityResolutionAvoided: input.phase === 'pre-resolution',})+ // Earlier buckets that admitted this request wrote their counts; a 503+ // charged no budget, so it carries none of them.+ input.res.removeHeader('X-RateLimit-Limit')+ input.res.removeHeader('X-RateLimit-Remaining')+ input.res.removeHeader('X-RateLimit-Reset')input.res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))input.res.setHeader('Cache-Control', 'no-store')input.res.status(503).json({diff --git a/control-api/src/middleware/externalSessionAuth.ts b/control-api/src/middleware/externalSessionAuth.tsindex 3cf4a859f..abf10ae32 100644--- a/control-api/src/middleware/externalSessionAuth.ts+++ b/control-api/src/middleware/externalSessionAuth.ts@@ -13,15 +13,30 @@ export const SESSION_BACKEND_RETRY_AFTER_SECONDS = 2// 57014 statement timeout), 58 system error.const BACKEND_UNAVAILABLE_SQLSTATE_CLASSES = new Set(['08', '53', '57', '58'])+// A Node system error code (ECONNREFUSED, ETIMEDOUT, EPIPE, ...). Node's own+// argument and state errors use ERR_* codes, which this does not match.+const SYSTEM_ERROR_CODE = /^E[A-Z]+$/+/*** True when the lookup failed because PostgreSQL could not be reached or could- * not run it. Pool acquire timeouts and socket failures are plain Errors with- * no SQLSTATE. A DatabaseError in any other class (22P02, 42P01, ...) is a- * defect in the query or the schema, which a 503 would hide.+ * not run it:+ * - a DatabaseError whose SQLSTATE class is 08, 53, 57 or 58;+ * - a plain Error, which is how pg and pg-pool report an acquire timeout or a+ * dropped connection;+ * - an error carrying a Node system code, as socket failures do (including the+ * AggregateError Node raises when every address of a host refuses).+ * Anything else is a defect: a DatabaseError in another class (22P02, 42P01,+ * ...) is in the query or the schema, and a TypeError or RangeError is in this+ * process's code. A 503 would hide either one.*/function isBackendUnavailableError(error: unknown): boolean {- if (!(error instanceof DatabaseError)) return true- return BACKEND_UNAVAILABLE_SQLSTATE_CLASSES.has(String(error.code).slice(0, 2))+ if (error instanceof DatabaseError) {+ return BACKEND_UNAVAILABLE_SQLSTATE_CLASSES.has(String(error.code).slice(0, 2))+ }+ if (!(error instanceof Error)) return false+ const code = (error as { code?: unknown }).code+ if (typeof code === 'string' && SYSTEM_ERROR_CODE.test(code)) return true+ return error.constructor === Error}export type ExternalAuthedRequest = Request & {diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.tsindex 943e36659..d7e285eb5 100644--- a/control-api/src/middleware/gfsUploadAdmission.ts+++ b/control-api/src/middleware/gfsUploadAdmission.ts@@ -108,7 +108,11 @@ function retryAfterSeconds(check: RateLimitCheck): number {function sendBackendUnavailable(res: Response): void {res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))- res.status(503).json({ error: 'gfs_upload_admission_unavailable' })+ res.setHeader('Cache-Control', 'no-store')+ res.status(503).json({+ error: 'gfs_upload_admission_unavailable',+ retryAfterSeconds: RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,+ })}function sendRateLimited(res: Response, limit: string, retryAfter: number, max: number): void {diff --git a/control-api/src/middleware/processMemoryRateLimiter.ts b/control-api/src/middleware/processMemoryRateLimiter.tsindex 86c69b992..5eb9d05bf 100644--- a/control-api/src/middleware/processMemoryRateLimiter.ts+++ b/control-api/src/middleware/processMemoryRateLimiter.ts@@ -23,6 +23,12 @@ export type ProcessMemoryDecision =* `maxTrackedKeys`. The store keeps a key until its cleanup timer has run* twice without a hit on it, so it can hold a few times `maxTrackedKeys`* entries at once; the cap bounds how fast that grows.+ *+ * The set's window is this class's own and does not line up with each key's+ * window in the store. A key whose store window is still running is always+ * admitted, so clearing the set cannot turn a key that is already being+ * counted into a `key_cap_reached` refusal. Such a key was admitted under the+ * cap when its window opened, so this does not raise the bound.*/export class ProcessMemoryRateLimiter {private readonly store = new MemoryStore()@@ -45,7 +51,12 @@ export class ProcessMemoryRateLimiter {this.windowStartMs = now}if (!this.keysThisWindow.has(key)) {- if (this.keysThisWindow.size >= this.maxTrackedKeys) return { outcome: 'key_cap_reached' }+ if (+ this.keysThisWindow.size >= this.maxTrackedKeys &&+ !(await this.hasLiveWindow(key, now))+ ) {+ return { outcome: 'key_cap_reached' }+ }this.keysThisWindow.add(key)}@@ -59,4 +70,9 @@ export class ProcessMemoryRateLimiter {resetMs: resetTime.getTime(),}}++ private async hasLiveWindow(key: string, now: number): Promise<boolean> {+ const client = await this.store.get(key)+ return client?.resetTime !== undefined && client.resetTime.getTime() > now+ }}diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.tsindex 0bcd39d94..fca6bc1c3 100644--- a/control-api/src/middleware/rateLimitMiddleware.ts+++ b/control-api/src/middleware/rateLimitMiddleware.ts@@ -2,12 +2,37 @@ import { NextFunction, Request, Response } from 'express'import { createHash } from 'node:crypto'import { LogThrottle } from '../observability/logThrottle.js'import { rateLimitHitsTotal } from '../observability/metrics.js'+import { boundedBucketKey } from '../services/rateLimitBucketKey.js'import {RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS,checkAndIncrement,} from '../services/rateLimiterService.js'import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'+/** What a limiter does when its Postgres backend cannot count a request. */+export type RateLimitBackendUnavailableMode = 'process-memory' | 'closed'++// Compile-time guard, checked by the CI typecheck of src/. The `'open'` mode+// (let the request through uncounted) was removed; adding it back to the union+// makes this alias fail to compile (TS2344).+type AssertTrue<T extends true> = T+export type OpenIsNotABackendUnavailableMode = AssertTrue<+ 'open' extends RateLimitBackendUnavailableMode ? false : true+>++export type RateLimitEnforcerOptions = {+ bucketType: string+ maxPerMinute: number+ onBackendUnavailable: RateLimitBackendUnavailableMode+}++/**+ * Charges one request to `key` and answers it when it must not proceed.+ * Resolves `true` when the caller may continue (the `X-RateLimit-*` headers are+ * set), or `false` after a 429 or 503 has been sent.+ */+export type RateLimitEnforcer = (req: Request, res: Response, key: string) => Promise<boolean>+/*** Factory for a per-request rate limit middleware backed by a PG token bucket.*@@ -31,6 +56,9 @@ import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'* - If `getBucketKey` returns `null` (e.g. unable to derive key), the* request is *allowed* but not counted — fail-open. Callers that need* strict enforcement should return a sentinel key instead.+ * - A key longer than 512 UTF-8 bytes is counted under its SHA-256+ * (`boundedBucketKey`), so a client-supplied value cannot grow either+ * backend's storage per request.* - When the limiter backend cannot count the request* (`backendAvailable: false`: the limiter pool is saturated, Postgres* returned an error, or the upsert returned no row), `onBackendUnavailable`@@ -41,19 +69,49 @@ import { ProcessMemoryRateLimiter } from './processMemoryRateLimiter.js'* key. Over the limit it answers 429 like a Postgres denial, and the* warn line carries `source: 'process-memory'`. The in-memory counter* is used only for requests Postgres could not count; the two counts- * are never added together. It tracks at most 100 000 keys per 60 s;+ * are never added together. So when an outage ends inside a window, a+ * key can get up to 2 × `maxPerMinute` in that window: up to the limit+ * counted in memory during the outage, then up to the limit counted in+ * Postgres after it. An upsert that commits but whose reply is lost is+ * counted by both, which only denies sooner. It tracks at most 100 000+ * keys per 60 s;* a new key beyond that answers 503 as `'closed'` does. The counter* lives in one process: control-api runs one replica today, so the* in-memory limit holds for the whole service only while that is* true. With N replicas a caller can get N × `maxPerMinute` requests* per minute while Postgres is unavailable.+ * - A 503 carries no `X-RateLimit-*` headers, including ones an earlier+ * limiter on the same route set, because no count backs them.*/-export function rateLimitMiddleware(opts: {- bucketType: string- maxPerMinute: number- getBucketKey: (req: Request) => string | null- onBackendUnavailable: 'process-memory' | 'closed'-}) {+export function rateLimitMiddleware(+ opts: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null }+) {+ const enforce = createRateLimitEnforcer(opts)+ return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {+ void (async () => {+ try {+ const key = opts.getBucketKey(req)+ if (!key) {+ // Fail-open: no key means we cannot attribute the call (usually a+ // pre-auth path); let it through rather than 500.+ next()+ return+ }+ if (await enforce(req, res, key)) next()+ } catch (err) {+ next(err)+ }+ })()+ }+}++/**+ * The limiter behind `rateLimitMiddleware`, for a route that can only derive+ * its key inside the handler (after authenticating the caller). Create it once+ * per route set, not per request: with `'process-memory'` it owns the+ * in-memory counter. Same semantics as `rateLimitMiddleware`.+ */+export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLimitEnforcer {// While the backend is down every request on this bucket fails the same way,// so the warn line is written once per minute; the counter records each one.const unavailableLogThrottle = new LogThrottle(60_000)@@ -71,6 +129,9 @@ export function rateLimitMiddleware(opts: {'rate limit backend unavailable')}+ res.removeHeader('X-RateLimit-Limit')+ res.removeHeader('X-RateLimit-Remaining')+ res.removeHeader('X-RateLimit-Reset')res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))res.setHeader('Cache-Control', 'no-store')res.status(503).json({@@ -109,58 +170,44 @@ export function rateLimitMiddleware(opts: {res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })}- return function rateLimitMw(req: Request, res: Response, next: NextFunction): void {- void (async () => {- try {- const key = opts.getBucketKey(req)- if (!key) {- // Fail-open: no key means we cannot attribute the call (usually a- // pre-auth path); let it through rather than 500.- next()- return- }-- const result = await checkAndIncrement(key, opts.maxPerMinute)- if (!result.backendAvailable) {- if (processMemory === null) {- answerUnavailable(req, res)- return- }- const decision = await processMemory.hit(key)- if (decision.outcome === 'key_cap_reached') {- answerUnavailable(req, res)- return- }- if (decision.outcome === 'denied') {- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)- answerDenied(req, res, key, {- count: decision.totalHits,- resetMs: decision.resetMs,- source: 'process-memory',- })- return- }- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))- res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))- next()- return- }- if (!result.allowed) {- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)- answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })- return- }-- rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', String(result.remaining))- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))- next()- } catch (err) {- next(err)+ return async function enforce(req: Request, res: Response, rawKey: string): Promise<boolean> {+ const key = boundedBucketKey(rawKey)+ const result = await checkAndIncrement(key, opts.maxPerMinute)+ if (!result.backendAvailable) {+ if (processMemory === null) {+ answerUnavailable(req, res)+ return false}- })()+ const decision = await processMemory.hit(key)+ if (decision.outcome === 'key_cap_reached') {+ answerUnavailable(req, res)+ return false+ }+ if (decision.outcome === 'denied') {+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_denied' }, 1)+ answerDenied(req, res, key, {+ count: decision.totalHits,+ resetMs: decision.resetMs,+ source: 'process-memory',+ })+ return false+ }+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))+ res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))+ res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))+ return true+ }+ if (!result.allowed) {+ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'denied' }, 1)+ answerDenied(req, res, key, { count: result.count, resetMs: result.resetMs })+ return false+ }++ rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)+ res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))+ res.setHeader('X-RateLimit-Remaining', String(result.remaining))+ res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))+ return true}}diff --git a/control-api/src/observability/logThrottle.ts b/control-api/src/observability/logThrottle.tsindex d497c19c5..b9168d489 100644--- a/control-api/src/observability/logThrottle.ts+++ b/control-api/src/observability/logThrottle.ts@@ -1,3 +1,6 @@+/** Distinct keys one throttle tracks before new keys share a single entry. */+export const LOG_THROTTLE_MAX_KEYS = 1_000+/*** Bounds how many lines one event writes: at most one per key per interval.* A suppressed event is only counted, and the next admitted line for that key@@ -9,6 +12,12 @@* interval, so the map is bounded by the keys active in the last two intervals.* A count still pending when its key is swept is lost from the log, not from* the metrics.+ *+ * Keys often come from clients, and a client can send a new one on every+ * request. Once `maxKeys` keys are tracked, every key not already tracked shares+ * one overflow entry: together they get at most one line per interval, and that+ * line carries their combined suppressed count. So one interval writes at most+ * `maxKeys + 1` lines, and the map holds at most `maxKeys + 1` entries.*/export class LogThrottle {private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>()@@ -16,9 +25,13 @@ export class LogThrottle {constructor(private readonly intervalMs: number,- private readonly now: () => number = () => Date.now()+ private readonly now: () => number = () => Date.now(),+ private readonly maxKeys: number = LOG_THROTTLE_MAX_KEYS) {if (!(intervalMs > 0)) throw new Error('LogThrottle: intervalMs must be > 0')+ if (!Number.isSafeInteger(maxKeys) || maxKeys < 1) {+ throw new Error('LogThrottle: maxKeys must be a positive integer')+ }this.lastSweep = now()}@@ -35,12 +48,14 @@ export class LogThrottle {admit(key: string): number | undefined {const now = this.now()this.sweep(now)- const entry = this.entries.get(key)+ const trackedKey =+ this.entries.has(key) || this.entries.size < this.maxKeys ? key : OVERFLOW_KEY+ const entry = this.entries.get(trackedKey)if (entry && now - entry.emittedAt < this.intervalMs) {entry.suppressed += 1return undefined}- this.entries.set(key, { emittedAt: now, suppressed: 0 })+ this.entries.set(trackedKey, { emittedAt: now, suppressed: 0 })return entry?.suppressed ?? 0}@@ -52,3 +67,7 @@ export class LogThrottle {this.lastSweep = now}}++// A NUL byte cannot appear in a key built from a header, a URL or a hex hash,+// so the overflow entry cannot collide with a real key.+const OVERFLOW_KEY = '\u0000overflow'diff --git a/control-api/src/routes/admin/auth.ts b/control-api/src/routes/admin/auth.tsindex c40efc41b..a02f7047f 100644--- a/control-api/src/routes/admin/auth.ts+++ b/control-api/src/routes/admin/auth.ts@@ -1,6 +1,7 @@import { type Request, type RequestHandler, Router } from 'express'import bcrypt from 'bcryptjs'import { ipKeyGenerator } from 'express-rate-limit'+import { createHash } from 'node:crypto'import { config } from '../../config.js'import { pool } from '../../db.js'import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/controlUIAuth.js'@@ -79,7 +80,11 @@ function publicAdminTokenRateLimits(routeName: string): [RequestHandler, Request.trim().toLowerCase()const login = String(req.body?.username || req.body?.login || '').trim()- return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${email || login || token.slice(0, 48) || 'missing'}`+ // The submitted value has no length bound before this point, and the key+ // is stored per window, so it enters the key as a fixed-size digest. The+ // digest also keeps emails out of the bucket table.+ const submitted = email || login || token.slice(0, 48) || 'missing'+ return `control-admin-public:${routeName}:${remoteAddressOf(req)}:${createHash('sha256').update(submitted).digest('hex')}`},onBackendUnavailable: 'process-memory',})diff --git a/control-api/src/routes/external/auth.ts b/control-api/src/routes/external/auth.tsindex e8dfb0aa8..48dd8690d 100644--- a/control-api/src/routes/external/auth.ts+++ b/control-api/src/routes/external/auth.ts@@ -1,4 +1,5 @@import { Router } from 'express'+import { createHash } from 'node:crypto'import { pool } from '../../db.js'import type { K8sGateway } from '../../k8s.js'import { isCurrentExternalSession } from '../../middleware/externalSessionAuth.js'@@ -127,7 +128,12 @@ export function createExternalAuthRouter(gateway: K8sGateway): Router {const email = String(req.body?.email || '').trim().toLowerCase()- return email ? `profile_password_reset:${email}` : `profile_password_reset_ip:${req.ip}`+ // The email is client-supplied and unbounded before this limiter, so it+ // enters the key as a fixed-size digest: an oversized value cannot grow+ // the limiter's storage, and the same email still maps to one bucket.+ return email+ ? `profile_password_reset:${createHash('sha256').update(email).digest('hex')}`+ : `profile_password_reset_ip:${req.ip}`},onBackendUnavailable: 'process-memory',}),diff --git a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts b/control-api/src/routes/mcp-host/user-approval-requests.routes.tsindex 948d05d55..288e345d5 100644--- a/control-api/src/routes/mcp-host/user-approval-requests.routes.ts+++ b/control-api/src/routes/mcp-host/user-approval-requests.routes.ts@@ -3,15 +3,16 @@ import { config } from '../../config.js'import type { K8sGateway } from '../../k8s.js'import { mcpHostHttpMetrics } from '../../middleware/mcpHostHttpMetrics.js'import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'-import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'-import { rateLimitHitsTotal } from '../../observability/metrics.js'+import {+ createRateLimitEnforcer,+ rateLimitMiddleware,+} from '../../middleware/rateLimitMiddleware.js'import {acknowledgeNotificationDelivery,claimNotificationDeliveries,failNotificationDelivery,resolvePendingWorkflowApprovalDelivery,} from '../../services/notificationDeliveryQueueService.js'-import { checkAndIncrement } from '../../services/rateLimiterService.js'import type { DirectRunAttributionBindingService } from '../../services/tracing/directRunAttributionBindingService.js'import {InvalidWorkflowApprovalRunBindingError,@@ -436,47 +437,23 @@ function classifyMcpHostCaller(: 'first-party-mcp-host'}-async function enforceMediumResolveRateLimit(- req: Request,- res: Response,- callerKey: string-): Promise<boolean> {- const bucketType = 'mcp_host_workflow_approval_medium_resolve'- const bucketKey = `medium-resolve:${callerKey}`- const result = await checkAndIncrement(bucketKey, config.approvalRlRequestPerMin)- if (!result.allowed) {- rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'denied' }, 1)- const retryAfterSec = Math.max(1, Math.ceil((result.resetMs - Date.now()) / 1000))- res.setHeader('Retry-After', String(retryAfterSec))- res.setHeader('X-RateLimit-Limit', String(config.approvalRlRequestPerMin))- res.setHeader('X-RateLimit-Remaining', '0')- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))- req.log?.warn(- {- event: 'rate_limit_denied',- bucketType,- bucketKey,- count: result.count,- maxPerMinute: config.approvalRlRequestPerMin,- },- 'rate limit exceeded'- )- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })- return false- }-- rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'allowed' }, 1)- res.setHeader('X-RateLimit-Limit', String(config.approvalRlRequestPerMin))- res.setHeader('X-RateLimit-Remaining', String(result.remaining))- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))- return true-}-export function createUserApprovalRequestsRoutes(gateway: K8sGateway,directRunAttributionBindingService: DirectRunAttributionBindingService): Router {const router = Router()+ // The caller key is known only after the handler authenticates the caller,+ // so these routes use the enforcer instead of the middleware. One instance+ // serves all four routes: they share the `medium-resolve:<caller>` bucket.+ // A 503 would fail the approval a user is waiting on, so while Postgres+ // cannot count, the limit is counted in this process's memory.+ const enforceMediumResolveRateLimit = createRateLimitEnforcer({+ bucketType: 'mcp_host_workflow_approval_medium_resolve',+ maxPerMinute: config.approvalRlRequestPerMin,+ onBackendUnavailable: 'process-memory',+ })+ const enforceMediumResolve = (req: Request, res: Response, callerKey: string) =>+ enforceMediumResolveRateLimit(req, res, `medium-resolve:${callerKey}`)router.post('/workflow-approvals/request',@@ -866,7 +843,7 @@ export function createUserApprovalRequestsRoutes(if (!caller) returnif (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) returnconst callerKey = getMcpHostCallerKey(caller.claims)- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return+ if (!(await enforceMediumResolve(req, res, callerKey))) returnconst parsedIdentity = parseProviderIdentityFromBody(req.body)if (!parsedIdentity.ok) {@@ -989,7 +966,7 @@ export function createUserApprovalRequestsRoutes(if (!caller) returnif (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) returnconst callerKey = getMcpHostCallerKey(caller.claims)- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return+ if (!(await enforceMediumResolve(req, res, callerKey))) returnconst deliveries = await claimNotificationDeliveries({medium: req.query.medium,@@ -1287,7 +1264,7 @@ export function createUserApprovalRequestsRoutes(if (!caller) returnif (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) returnconst callerKey = getMcpHostCallerKey(caller.claims)- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return+ if (!(await enforceMediumResolve(req, res, callerKey))) returnconst body = requestBodyRecord(req.body)const code = String(body.code || '').trim()@@ -1337,7 +1314,7 @@ export function createUserApprovalRequestsRoutes(if (!caller) returnif (!requireMcpHostControlScope(caller, res, 'workflow:approval:resolve')) returnconst callerKey = getMcpHostCallerKey(caller.claims)- if (!(await enforceMediumResolveRateLimit(req, res, callerKey))) return+ if (!(await enforceMediumResolve(req, res, callerKey))) returnconst parsedIdentity = parseProviderIdentityFromBody(req.body)if (!parsedIdentity.ok) {diff --git a/control-api/src/services/rateLimitBucketKey.ts b/control-api/src/services/rateLimitBucketKey.tsnew file mode 100644index 000000000..847a0daf5--- /dev/null+++ b/control-api/src/services/rateLimitBucketKey.ts@@ -0,0 +1,24 @@+import { createHash } from 'node:crypto'++/** Longest bucket key, in UTF-8 bytes, that the rate limiters store as given. */+export const MAX_RATE_LIMIT_BUCKET_KEY_BYTES = 512++const LONG_KEY_PREFIX = 'sha256-long-key:'++/**+ * The form of a bucket key that both limiter backends store.+ *+ * Several key builders include a value the client submits (an email, a login,+ * a token prefix, a route parameter). Without a bound, an unauthenticated+ * client can send one oversized value per request: Postgres rejects the upsert+ * (the btree entry limit is 2704 bytes), so the request falls through to the+ * process-memory counter, which keeps each key for its whole window. A key over+ * `MAX_RATE_LIMIT_BUCKET_KEY_BYTES` is therefore replaced by its SHA-256. The+ * same input always gives the same stored key, so an oversized value is still+ * counted and limited like any other; only its size changes. Keys at or under+ * the bound are returned unchanged, so existing buckets keep their keys.+ */+export function boundedBucketKey(key: string): string {+ if (Buffer.byteLength(key, 'utf8') <= MAX_RATE_LIMIT_BUCKET_KEY_BYTES) return key+ return `${LONG_KEY_PREFIX}${createHash('sha256').update(key).digest('hex')}`+}diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex ba2adc912..5ee268d78 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -4,6 +4,7 @@ import { pool, rateLimitPool } from '../db.js'import { LogThrottle } from '../observability/logThrottle.js'import { rootLogger } from '../observability/logger.js'import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'+import { boundedBucketKey } from './rateLimitBucketKey.js'/*** PG-backed sliding-window-ish rate limiter.@@ -22,7 +23,8 @@ import { rateLimitBackendErrorsTotal } from '../observability/metrics.js'const WINDOW_MS = 60_000-// One rate_limit_db_error line per bucket per window.+// One rate_limit_db_error line per bucket key per window. Keys come from+// clients, so the throttle also caps how many distinct keys it tracks.const dbErrorLogThrottle = new LogThrottle(WINDOW_MS)export type RateLimitCheck = {@@ -87,6 +89,9 @@ export async function checkAndIncrementWithQuery(if (!Number.isSafeInteger(cost) || cost < 1) throw new Error('rate limit cost must be positive')const windowStartMs = currentWindowStartMs(nowMs)const resetMs = windowStartMs + WINDOW_MS+ // Callers that build keys from client input without calling the middleware+ // get the same bound; an already bounded key is returned unchanged.+ const storedKey = boundedBucketKey(bucketKey)let result: { rows: unknown[] } | null | undefinedtry {@@ -96,7 +101,7 @@ export async function checkAndIncrementWithQuery(ON CONFLICT (bucket_key, window_start_ms) DO UPDATESET count = rate_limit_buckets.count + EXCLUDED.countRETURNING count`,- [bucketKey, windowStartMs, cost]+ [storedKey, windowStartMs, cost])} catch (err) {// The service does not decide admission. It reports backendAvailable:false@@ -107,10 +112,10 @@ export async function checkAndIncrementWithQuery(// the external GFS limiter logs as hashedKey, so the lines still join. It// is a correlation id, not anonymization: unsalted, so a low-entropy key// (an IP, an email) can be recovered by guessing. While the backend is- // down every request lands here, so the line is throttled per bucket and- // the counter keeps the full count.+ // down every request lands here, so the line is throttled per bucket key+ // and the counter keeps the full count.rateLimitBackendErrorsTotal.inc()- const hashedKey = createHash('sha256').update(bucketKey).digest('hex')+ const hashedKey = createHash('sha256').update(storedKey).digest('hex')const suppressed = dbErrorLogThrottle.admit(hashedKey)if (suppressed !== undefined) {rootLogger.warn(diff --git a/control-api/test/app.memberRegistrationUnavailable.test.ts b/control-api/test/app.memberRegistrationUnavailable.test.tsindex 099ef9dd0..b163c8c1e 100644--- a/control-api/test/app.memberRegistrationUnavailable.test.ts+++ b/control-api/test/app.memberRegistrationUnavailable.test.ts@@ -18,6 +18,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({rateLimitMiddleware:() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>next(),+ createRateLimitEnforcer: () => async () => true,}))describe('app: member-registration unavailability maps to 503 (real error middleware)', () => {diff --git a/control-api/test/app.publishImageRefIdentity.test.ts b/control-api/test/app.publishImageRefIdentity.test.tsindex 1b6bb8f6b..bec070aed 100644--- a/control-api/test/app.publishImageRefIdentity.test.ts+++ b/control-api/test/app.publishImageRefIdentity.test.ts@@ -38,6 +38,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({rateLimitMiddleware:() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>next(),+ createRateLimitEnforcer: () => async () => true,}))const baseBody = (imageRef: string) => ({diff --git a/control-api/test/app.publisherUiEnabled.test.ts b/control-api/test/app.publisherUiEnabled.test.tsindex 7d2f68854..24a57eaca 100644--- a/control-api/test/app.publisherUiEnabled.test.ts+++ b/control-api/test/app.publisherUiEnabled.test.ts@@ -40,6 +40,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({rateLimitMiddleware:() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>next(),+ createRateLimitEnforcer: () => async () => true,}))describe('GET /admin/registry/publish-scope: publisherUiEnabled (effective, through the route)', () => {diff --git a/control-api/test/app.registryCatalogUnavailable.test.ts b/control-api/test/app.registryCatalogUnavailable.test.tsindex 3cd09a987..5108e7b83 100644--- a/control-api/test/app.registryCatalogUnavailable.test.ts+++ b/control-api/test/app.registryCatalogUnavailable.test.ts@@ -39,6 +39,7 @@ vi.mock('../src/middleware/rateLimitMiddleware.js', () => ({rateLimitMiddleware:() => (_req: express.Request, _res: express.Response, next: express.NextFunction) =>next(),+ createRateLimitEnforcer: () => async () => true,}))describe('app: registry-unavailable forwarding at the global error handler (real middleware)', () => {diff --git a/control-api/test/db.poolFactory.test.ts b/control-api/test/db.poolFactory.test.tsindex d2ca8309b..893287ead 100644--- a/control-api/test/db.poolFactory.test.ts+++ b/control-api/test/db.poolFactory.test.ts@@ -109,6 +109,26 @@ describe('rate limiter Postgres pool', () => {})})+ it.each([+ ['CORE_POOL_MAX', ' ', 'max', 10, 0],+ ['CORE_POOL_MAX', '', 'max', 10, 0],+ ['RATE_LIMIT_POOL_MAX', '\t', 'max', 6, 1],+ ['RATE_LIMIT_POOL_STATEMENT_TIMEOUT_MS', ' ', 'statement_timeout', 3_000, 1],+ ] as const)(+ 'treats a blank %s (%j) as unset, as config.ts does, and uses the default',+ async (name, value, field, expected, poolIndex) => {+ process.env[name] = value+ try {+ await import('../src/db.js')+ // Witness: module load built both pools; the one reading the value got the default.+ expect(fakePools).toHaveLength(2)+ expect(fakePools[poolIndex]?.config[field]).toBe(expected)+ } finally {+ delete process.env[name]+ }+ }+ )+it.each([// [env, value, range, pools built before the refusal (core is built first)]['RATE_LIMIT_POOL_MAX', '17', '[1, 16]', 1],diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.tsindex adc8669fd..821fb7f5b 100644--- a/control-api/test/gfsUploadAdmission.test.ts+++ b/control-api/test/gfsUploadAdmission.test.ts@@ -280,8 +280,12 @@ describe('GFS upload admission', () => {checkMock.mockResolvedValue(rateCheck({ backendAvailable: false }))const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')expect(response.status).toBe(503)- expect(response.body).toEqual({ error: 'gfs_upload_admission_unavailable' })+ expect(response.body).toEqual({+ error: 'gfs_upload_admission_unavailable',+ retryAfterSeconds: 2,+ })expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['cache-control']).toBe('no-store')expect(checkMock).toHaveBeenCalled()expect(acquireMock).not.toHaveBeenCalled()})diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.tsindex 9406e3cbf..fcd2728b4 100644--- a/control-api/test/middleware.externalGfsRateLimit.test.ts+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts@@ -304,6 +304,11 @@ describe('external GFS rate boundary', () => {expect(response.body).toEqual({ error: 'gfs_rate_limit_unavailable', retryAfterSeconds: 2 })expect(response.headers['retry-after']).toBe('2')expect(response.headers['cache-control']).toBe('no-store')+ // The three buckets that admitted the request wrote X-RateLimit-*; the 503+ // does not carry their counts, which describe a budget it did not charge.+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()+ expect(response.headers['x-ratelimit-reset']).toBeUndefined()expect(checkAndIncrement).toHaveBeenCalledTimes(4)expect(checkAndIncrement).toHaveBeenLastCalledWith(`gfs-ext:resolved:resource:actor:linked-admin:${CONTROL_ADMIN_ID}`,diff --git a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts b/control-api/test/middleware.externalSessionAuth.lifecycle.test.tsindex 0138cbcb9..95749bfa2 100644--- a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts+++ b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts@@ -120,6 +120,11 @@ describe('external session lifecycle gate', () => {],// Five characters, like a SQLSTATE, but a socket error: no SQLSTATE class.['a broken pipe', Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })],+ // Node raises this when every address of a multi-address host refuses.+ [+ 'a refused socket on every address',+ Object.assign(new AggregateError([], 'connect ECONNREFUSED'), { code: 'ECONNREFUSED' }),+ ],['SQLSTATE 08006 connection_failure', databaseError('08006')],['SQLSTATE 53300 too_many_connections', databaseError('53300')],['SQLSTATE 57P01 admin_shutdown', databaseError('57P01')],@@ -138,6 +143,15 @@ describe('external session lifecycle gate', () => {['SQLSTATE 22P02 invalid_text_representation', databaseError('22P02')],['SQLSTATE 42P01 undefined_table', databaseError('42P01')],['SQLSTATE 42703 undefined_column', databaseError('42703')],+ // Defects in this process's code, not in the connection to PostgreSQL.+ ['a TypeError', new TypeError("Cannot read properties of undefined (reading 'query')")],+ ['a RangeError', new RangeError('Invalid array length')],+ [+ 'a Node argument error',+ Object.assign(new TypeError('The "string" argument must be of type string'), {+ code: 'ERR_INVALID_ARG_TYPE',+ }),+ ],])('passes %s to the error handler as a 500, not a 503', async (_label, error) => {verifyToken.mockReturnValueOnce(claims)poolQuery.mockRejectedValueOnce(error)diff --git a/control-api/test/middleware.processMemoryRateLimiter.test.ts b/control-api/test/middleware.processMemoryRateLimiter.test.tsindex 0cd45b10c..a025a154c 100644--- a/control-api/test/middleware.processMemoryRateLimiter.test.ts+++ b/control-api/test/middleware.processMemoryRateLimiter.test.ts@@ -46,6 +46,32 @@ describe('ProcessMemoryRateLimiter', () => {expect(await limiter.hit('c')).toMatchObject({ outcome: 'allowed', totalHits: 1 })})+ it('keeps counting a key whose window is still running after the tracked-key set is cleared', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(START_MS)+ const limiter = new ProcessMemoryRateLimiter(5, 2)++ // 'b' opens its store window at +30 s, so it runs until +90 s.+ vi.setSystemTime(START_MS + 30_000)+ expect(await limiter.hit('b')).toMatchObject({ outcome: 'allowed', totalHits: 1 })++ // At +60 s the class's own window rolls over and its key set is cleared;+ // two new keys fill it to the cap.+ vi.setSystemTime(START_MS + 60_000)+ expect((await limiter.hit('x')).outcome).toBe('allowed')+ expect((await limiter.hit('y')).outcome).toBe('allowed')++ vi.setSystemTime(START_MS + 61_000)+ // Witness: the cap is in force for a key with no running window.+ expect(await limiter.hit('z')).toEqual({ outcome: 'key_cap_reached' })+ // 'b' is still inside its window, so it is counted, not refused.+ expect(await limiter.hit('b')).toEqual({+ outcome: 'allowed',+ totalHits: 2,+ resetMs: START_MS + 90_000,+ })+ })+it('tracks at most 100 000 keys by default', async () => {expect(PROCESS_MEMORY_MAX_TRACKED_KEYS).toBe(100_000)const limiter = new ProcessMemoryRateLimiter(1)diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.tsindex 03f1b68e6..d096a78de 100644--- a/control-api/test/middleware.rateLimitMiddleware.test.ts+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts@@ -42,18 +42,6 @@ function unavailable(): RateLimitCheck {}}-// Compile-time check only. control-api's tsconfig covers src/, so vitest does-// not typecheck this file; run tsc over it to verify the expectation below.-export function buildWithRemovedOpenMode() {- return rateLimitMiddleware({- bucketType: 'unit_bucket',- maxPerMinute: LIMIT,- getBucketKey: () => 'unit:key',- // @ts-expect-error 'open' is no longer a mode- onBackendUnavailable: 'open',- })-}-function appWith(onBackendUnavailable: 'process-memory' | 'closed') {const handler = vi.fn((_req: express.Request, res: express.Response) => {res.status(204).end()@@ -232,6 +220,9 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {setHeader(name: string, value: string) {headers[name.toLowerCase()] = value},+ removeHeader(name: string) {+ delete headers[name.toLowerCase()]+ },status(code: number) {this.statusCode = codereturn this@@ -275,6 +266,88 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {])}, 60_000)+ it("'closed' 503 carries no X-RateLimit-* headers left by an earlier limiter on the route", async () => {+ // First limiter counts and admits; the second cannot reach its backend.+ checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(unavailable())+ const handler = vi.fn((_req: express.Request, res: express.Response) => res.status(204).end())+ const app = express()+ app.get(+ '/limited',+ (req, _res, next) => {+ ;(req as unknown as { log: { warn: () => void } }).log = { warn: () => {} }+ next()+ },+ rateLimitMiddleware({+ bucketType: 'first_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'first:key',+ onBackendUnavailable: 'closed',+ }),+ rateLimitMiddleware({+ bucketType: 'second_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'second:key',+ onBackendUnavailable: 'closed',+ }),+ handler+ )++ const response = await request(app).get('/limited')++ expect(response.status).toBe(503)+ // Witness: both limiters ran, the first one admitting the request.+ expect(checkAndIncrement.mock.calls).toEqual([+ ['first:key', LIMIT],+ ['second:key', LIMIT],+ ])+ expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()+ expect(response.headers['x-ratelimit-remaining']).toBeUndefined()+ expect(response.headers['x-ratelimit-reset']).toBeUndefined()+ expect(handler).not.toHaveBeenCalled()+ })++ it('bounds an oversized bucket key before it reaches either backend, and keeps it stable', async () => {+ const huge = `unit:${'x'.repeat(10_000)}`+ checkAndIncrement.mockResolvedValue(unavailable())+ const middleware = rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => huge,+ onBackendUnavailable: 'process-memory',+ })+ const increment = vi.spyOn(MemoryStore.prototype, 'increment')+ try {+ const app = express()+ app.get(+ '/limited',+ (req, _res, next) => {+ ;(req as unknown as { log: { warn: () => void } }).log = { warn: () => {} }+ next()+ },+ middleware,+ (_req, res) => res.status(204).end()+ )++ const first = await request(app).get('/limited')+ const second = await request(app).get('/limited')++ expect([first.status, second.status]).toEqual([204, 204])+ // Same key both times, so the in-memory count still accumulates.+ expect(second.headers['x-ratelimit-remaining']).toBe(String(LIMIT - 2))+ // Witness: the Postgres limiter and the in-memory store were both reached.+ expect(checkAndIncrement).toHaveBeenCalledTimes(2)+ expect(increment).toHaveBeenCalledTimes(2)+ const postgresKeys = checkAndIncrement.mock.calls.map(([key]) => key as string)+ const memoryKeys = increment.mock.calls.map(([key]) => key)+ const bounded = `sha256-long-key:${createHash('sha256').update(huge).digest('hex')}`+ expect(postgresKeys).toEqual([bounded, bounded])+ expect(memoryKeys).toEqual([bounded, bounded])+ } finally {+ increment.mockRestore()+ }+ })+it("'closed' still admits a counted request and still answers 429 over the limit", async () => {checkAndIncrement.mockResolvedValueOnce(available(1)).mockResolvedValueOnce(available(6))const { app, handler } = appWith('closed')diff --git a/control-api/test/observability.logThrottle.test.ts b/control-api/test/observability.logThrottle.test.tsindex ab8054cf6..28dbeffcf 100644--- a/control-api/test/observability.logThrottle.test.ts+++ b/control-api/test/observability.logThrottle.test.ts@@ -52,7 +52,39 @@ describe('LogThrottle', () => {expect(throttle.trackedKeyCount).toBe(1)})- it('rejects a non-positive interval', () => {+ it('sends every key past maxKeys to one shared overflow entry, so an interval writes at most maxKeys + 1 lines', () => {+ let t = 1_000+ const throttle = new LogThrottle(60_000, () => t, 3)++ const tracked = ['a', 'b', 'c'].map(key => throttle.admit(key))+ const overflow = Array.from({ length: 50 }, (_, i) => throttle.admit(`new-${i}`))++ // Witness: the three tracked keys each got their line.+ expect(tracked).toEqual([0, 0, 0])+ // Fifty distinct new keys share one line between them.+ expect(overflow.filter(r => r !== undefined)).toEqual([0])+ expect(throttle.trackedKeyCount).toBe(4)+ // A tracked key is still throttled on its own entry.+ expect(throttle.admit('a')).toBeUndefined()++ t += 60_000+ // The overflow line carries the combined count of the 49 it suppressed.+ expect(throttle.admit('new-999')).toBe(49)+ })++ it('tracks at most 1 000 keys by default', () => {+ const t = 1_000+ const throttle = new LogThrottle(60_000, () => t)+ const admitted = Array.from({ length: 5_000 }, (_, i) => throttle.admit(`k${i}`)).filter(+ r => r !== undefined+ )+ expect(admitted).toHaveLength(1_001)+ expect(throttle.trackedKeyCount).toBe(1_001)+ })++ it('rejects a non-positive interval and a non-positive key cap', () => {expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/)+ expect(() => new LogThrottle(1_000, () => 0, 0)).toThrow(/maxKeys must be a positive integer/)+ expect(() => new LogThrottle(1_000, () => 0, 1.5)).toThrow(/maxKeys must be a positive integer/)})})diff --git a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts b/control-api/test/routes.adminAuth.publicTokenLimiter.test.tsindex f22ec20f8..b5efad0b6 100644--- a/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts+++ b/control-api/test/routes.adminAuth.publicTokenLimiter.test.ts@@ -1,8 +1,11 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { createHash } from 'node:crypto'import request from 'supertest'import { config } from '../src/config.js'+const sha256 = (value: string) => createHash('sha256').update(value).digest('hex')+/*** The public control-admin token routes (password reset, invitation, email* confirmation) are called before login. Each route has two buckets:@@ -212,11 +215,37 @@ describe('public control-admin token limiter', () => {expect(limiterCalls).toEqual([{- key: `control-admin-public:${routeName}:203.0.113.7:[email protected]`,+ key: `control-admin-public:${routeName}:203.0.113.7:${sha256('[email protected]')}`,max: PER_VALUE,},{ key: `control-admin-public-ip:${routeName}:203.0.113.7`, max: PER_IP },])})++ it('keys the per-value bucket on a fixed-size digest of the submitted value, whatever its size', async () => {+ const { app } = buildApp()+ // Under express.json's default 100 kB limit here; production allows more.+ const huge = `${'a'.repeat(50_000)}@corp.test`+ await request(app)+ .post('/admin/auth/password-reset/request')+ .set('x-forwarded-for', '203.0.113.9')+ .send({ email: huge })+ await request(app)+ .post('/admin/auth/password-reset/request')+ .set('x-forwarded-for', '203.0.113.9')+ .send({ email: huge })++ // Liveness witness: both requests reached the per-value limiter.+ const perValueKeys = limiterCalls+ .map(call => call.key)+ .filter(key => key.startsWith('control-admin-public:'))+ expect(perValueKeys).toHaveLength(2)+ // Same value, same bucket; the key does not grow with the value.+ expect(perValueKeys[0]).toBe(perValueKeys[1])+ expect(perValueKeys[0]).toBe(+ `control-admin-public:password_reset_request:203.0.113.9:${sha256(huge)}`+ )+ expect(Buffer.byteLength(perValueKeys[0] as string)).toBeLessThan(200)+ })})diff --git a/control-api/test/routes.externalAuth.passwordResetLimiter.test.ts b/control-api/test/routes.externalAuth.passwordResetLimiter.test.tsnew file mode 100644index 000000000..ef95e3090--- /dev/null+++ b/control-api/test/routes.externalAuth.passwordResetLimiter.test.ts@@ -0,0 +1,91 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import { createHash } from 'node:crypto'+import request from 'supertest'++/**+ * `POST /external/auth/password-reset/request` is called before login and keys+ * its limiter on the submitted email. The email has no length bound before the+ * limiter, so it must enter the bucket key as a fixed-size digest: otherwise a+ * client can grow the limiter's storage with one oversized value per request.+ */++const limiterCalls = vi.hoisted(() => [] as Array<{ key: string; max: number }>)++vi.mock('../src/services/rateLimiterService.js', () => ({+ RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS: 2,+ checkAndIncrement: vi.fn(async (key: string, max: number) => {+ limiterCalls.push({ key, max })+ return {+ allowed: true,+ remaining: max - 1,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count: 1,+ backendAvailable: true,+ }+ }),+}))++const directory = vi.hoisted(() => ({+ getTeamAgents: vi.fn(),+ getUserAgents: vi.fn(),+ googleLoginData: vi.fn(),+ passwordLoginData: vi.fn(),+ requestProfilePasswordReset: vi.fn(async () => undefined),+}))+vi.mock('../src/services/directory/index.js', () => directory)+vi.mock('../src/db.js', () => ({ pool: { query: vi.fn() } }))++const { createExternalAuthRouter } = await import('../src/routes/external/auth.js')++const sha256 = (value: string) => createHash('sha256').update(value).digest('hex')++function buildApp() {+ const app = express()+ app.set('trust proxy', 1)+ app.use(express.json())+ app.use(createExternalAuthRouter({} as never))+ return app+}++describe('external password reset request limiter key', () => {+ beforeEach(() => {+ limiterCalls.length = 0+ directory.requestProfilePasswordReset.mockClear()+ })++ it('keys the bucket on a digest of the normalized email', async () => {+ const response = await request(buildApp())+ .post('/external/auth/password-reset/request')+ .send({ email: ' [email protected] ' })++ expect(response.status).toBe(200)+ // Liveness witness: the handler ran after the limiter.+ expect(directory.requestProfilePasswordReset).toHaveBeenCalledWith('[email protected]')+ expect(limiterCalls).toEqual([+ { key: `profile_password_reset:${sha256('[email protected]')}`, max: 5 },+ ])+ })++ it('gives an oversized email the same fixed-size key on every request', async () => {+ const app = buildApp()+ // Under express.json's default 100 kB limit here; production allows more.+ const huge = `${'a'.repeat(50_000)}@example.test`+ await request(app).post('/external/auth/password-reset/request').send({ email: huge })+ await request(app).post('/external/auth/password-reset/request').send({ email: huge })++ expect(limiterCalls).toHaveLength(2)+ expect(limiterCalls[0]?.key).toBe(`profile_password_reset:${sha256(huge)}`)+ expect(limiterCalls[1]?.key).toBe(limiterCalls[0]?.key)+ })++ it('keys a request with no email on the source address', async () => {+ await request(buildApp())+ .post('/external/auth/password-reset/request')+ .set('x-forwarded-for', '198.51.100.4')+ .send({})++ expect(limiterCalls).toEqual([{ key: 'profile_password_reset_ip:198.51.100.4', max: 5 }])+ })+})diff --git a/control-api/test/routes.mcp-host.user-approval-requests.test.ts b/control-api/test/routes.mcp-host.user-approval-requests.test.tsindex b76350597..192aeb533 100644--- a/control-api/test/routes.mcp-host.user-approval-requests.test.ts+++ b/control-api/test/routes.mcp-host.user-approval-requests.test.ts@@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'import jwt from 'jsonwebtoken'import request from 'supertest'import { createApp } from '../src/app.js'+import { config } from '../src/config.js'import * as notificationDeliveryQueueService from '../src/services/notificationDeliveryQueueService.js'import * as userApprovalRequestService from '../src/services/userApprovalRequestService.js'import * as workflowApprovalMediumLinkSessionService from '../src/services/workflowApprovalMediumLinkSessionService.js'@@ -1269,6 +1270,38 @@ describe('User Approval Request Routes', () => {).not.toHaveBeenCalled()})+ it('keeps counting in process memory when the Postgres limiter cannot count, and answers 429 over the limit', async () => {+ // The db mock has no rateLimitPool, so every checkAndIncrement reports+ // backendAvailable:false, as it does when the limiter pool is saturated.+ const previousLimit = config.approvalRlRequestPerMin+ config.approvalRlRequestPerMin = 3+ try {+ const limitedApp = createApp(gateway as never)+ const token = issueProviderDecisionControlToken(['workflow:approval:resolve'])+ const send = () =>+ request(limitedApp)+ .post('/api/v1/workflow-approval-mediums/resolve')+ .set('Authorization', `Bearer ${token}`)+ .send({ providerIdentity: { medium: 'telegram', providerUserId: '123456' } })++ const admitted = [await send(), await send(), await send()]+ const refused = await send()++ // Witness: the first three reached the handler's validation, each one+ // counted by the in-memory limiter.+ expect(admitted.map(r => r.status)).toEqual([400, 400, 400])+ expect(admitted.map(r => r.body.error)).toEqual(+ Array(3).fill('provider channel identity is required')+ )+ expect(admitted.map(r => r.headers['x-ratelimit-remaining'])).toEqual(['2', '1', '0'])+ expect(refused.status).toBe(429)+ expect(refused.body).toMatchObject({ error: 'Too Many Requests' })+ expect(refused.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)+ } finally {+ config.approvalRlRequestPerMin = previousLimit+ }+ })+it('requires stable provider channel identity', async () => {const token = issueProviderDecisionControlToken(['workflow:approval:resolve'])diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.tsindex 6a3928140..522a9c6d4 100644--- a/control-api/test/services.rateLimiter.test.ts+++ b/control-api/test/services.rateLimiter.test.ts@@ -208,7 +208,7 @@ describe('rateLimiterService', () => {await replacement.release()})- it('fails open when the limiter pool query throws (DB error)', async () => {+ it('reports the backend unavailable when the limiter pool query throws (DB error); the caller decides the policy', async () => {mockRateLimitPoolQuery.mockRejectedValueOnce(new Error('connection refused'))const r = await checkAndIncrement('test:bucket:failopen', 5)expect(r.allowed).toBe(true)@@ -271,7 +271,7 @@ describe('rateLimiterService', () => {}})- it('fails open when the limiter pool query returns empty rows', async () => {+ it('reports the backend unavailable when the limiter pool query returns empty rows', async () => {mockRateLimitPoolQuery.mockResolvedValueOnce({ rows: [], rowCount: 0 })const r = await checkAndIncrement('test:bucket:emptyrows', 5)expect(r.allowed).toBe(true)