Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): coordinate administrative quota enforcement
Alfredo Lopezb4f73fcf2485
diff --git a/charts/clerum-crds/crds/workflowrecipe.yaml b/charts/clerum-crds/crds/workflowrecipe.yamlindex b8449c761..a27c91bdf 100644--- a/charts/clerum-crds/crds/workflowrecipe.yaml+++ b/charts/clerum-crds/crds/workflowrecipe.yaml@@ -1795,7 +1795,7 @@ spec:maxInvocationsPerMinute:type: integerminimum: 1- description: Rate limit per workload; platform default 120 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348).+ description: Rate limit per workload; platform default 600 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348).resultRetentionHours:type: integerminimum: 1@@ -1842,7 +1842,7 @@ spec:maxNotificationsPerMinute:type: integerminimum: 1- description: Rate limit per workload; platform default 150 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348).+ description: Rate limit per workload; platform default 750 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348).maxTitleBytes:type: integerminimum: 1diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 6ad274302..6198436cc 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.707",+ "version": "0.1.708","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.707",+ "version": "0.1.708","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 32cb8e0a1..359b6ec2e 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.707",+ "version": "0.1.708","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex 4cda6e62b..f3af91b86 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -211,6 +211,27 @@ type Config = {adminSubscriptionReadPerMin: numberadminSubscriptionWritePerMin: numbersubscriptionOAuthCallbackPerMin: number+ adminWorkflowReadPerMin: number+ adminWorkflowGrantReadPerMin: number+ adminWorkflowGrantWritePerMin: number+ adminWorkflowTriggerPerMin: number+ adminOutputsReadPerMin: number+ llmProviderAttemptAuthorizePerMin: number+ llmProviderAttemptAuthorizeAnonymousIpPerMin: number+ adminRegistryKeysPerMin: number+ adminRegistryGrantsPerMin: number+ adminRegistryConnectStatusPerMin: number+ adminRegistryConnectRequestPerMin: number+ adminRegistryConnectRecoveryPerMin: number+ adminConnectorDeleteEdgePerMin: number+ adminConnectorDeletePerMin: number+ pluginSdkAdminRlPerMin: number+ pluginSdkInternalRlPerMin: number+ pluginSdkCredentialRlPerMin: number+ pluginSdkAuthenticatedPreauthRlPerMin: number+ adminGfsGrantsPerMin: number+ adminGfsSharesPerMin: number+ adminGfsLegacyGrantReportPerMin: numberapprovalRlRefreshPerMin: numberapprovalRlExternalPerMin: numberapprovalRlExternalEdgePerMin: number@@ -1016,8 +1037,71 @@ export const config: Config = {'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',100),- approvalRlRequestPerMin: Number(process.env.APPROVAL_RL_REQUEST_PER_MIN || 120),- approvalRlRefreshPerMin: Number(process.env.APPROVAL_RL_REFRESH_PER_MIN || 20),+ adminWorkflowReadPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN', 300),+ adminWorkflowGrantReadPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN',+ 300+ ),+ adminWorkflowGrantWritePerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN',+ 100+ ),+ adminWorkflowTriggerPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN',+ 50+ ),+ adminOutputsReadPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN', 150),+ llmProviderAttemptAuthorizePerMin: positiveIntegerFromEnv(+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN',+ 300+ ),+ llmProviderAttemptAuthorizeAnonymousIpPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN',+ 60+ ),+ adminRegistryKeysPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN', 150),+ adminRegistryGrantsPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN',+ 150+ ),+ adminRegistryConnectStatusPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN',+ 150+ ),+ adminRegistryConnectRequestPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN',+ 15+ ),+ adminRegistryConnectRecoveryPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN',+ 50+ ),+ adminConnectorDeleteEdgePerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN',+ 300+ ),+ adminConnectorDeletePerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN',+ 150+ ),+ pluginSdkAdminRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN', 600),+ pluginSdkInternalRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN', 600),+ pluginSdkCredentialRlPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',+ 1800+ ),+ pluginSdkAuthenticatedPreauthRlPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',+ 6000+ ),+ adminGfsGrantsPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN', 150),+ adminGfsSharesPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_GFS_SHARES_PER_MIN', 150),+ adminGfsLegacyGrantReportPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN',+ 150+ ),+ approvalRlRequestPerMin: positiveIntegerFromEnv('APPROVAL_RL_REQUEST_PER_MIN', 600),+ approvalRlRefreshPerMin: positiveIntegerFromEnv('APPROVAL_RL_REFRESH_PER_MIN', 100),approvalRlExternalPerMin: externalRateLimitConfig.operation,approvalRlExternalEdgePerMin: externalRateLimitConfig.session,// By default the source-IP backstop is wider than the per-session edge bucket@@ -1025,7 +1109,7 @@ export const config: Config = {// platform-wide ceiling. Operators may override these independent scopes;// crossed values are preserved after the boot advisory above.approvalRlExternalClientIpPerMin: externalRateLimitConfig.clientIp,- oauthBrokerRlPerMin: Number(process.env.CONTROL_API_OAUTH_BROKER_RL_PER_MIN || 60),+ oauthBrokerRlPerMin: positiveIntegerFromEnv('CONTROL_API_OAUTH_BROKER_RL_PER_MIN', 300),// The public control-admin token routes (password reset, invitation, email// confirmation) have two buckets per route. The first keys on source IP and// the submitted value (email, login or token prefix): one person retrying.@@ -1045,15 +1129,15 @@ export const config: Config = {// would effectively fail open.pluginSdkNotificationsRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN',- 150+ 750),pluginSdkPromptBridgeRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN',- 120+ 600),pluginSdkRequestBucketRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN',- 600+ 6000),pluginSdkPreauthRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN', 600),// External GFS authority-boundary budgets. The four keyed by source IP or@@ -1142,7 +1226,7 @@ export const config: Config = {// the single-hold mechanism volume and covers one held wake plus prewarm// bursts from a few concurrent devices (4 x 3 = 12) plus a real message.// Derivation is regression-guarded by test/config.hostWakeRateLimit.test.ts.- hostWakeRlPerMin: Number(process.env.CONTROL_API_HOST_WAKE_RL_PER_MIN || 30),+ hostWakeRlPerMin: positiveIntegerFromEnv('CONTROL_API_HOST_WAKE_RL_PER_MIN', 150),hostWakeCoalesceWindowMs: Number(process.env.CONTROL_API_HOST_WAKE_COALESCE_WINDOW_MS || 2000),approvalMediumChallengeTtlSec: Number(process.env.WORKFLOW_APPROVAL_MEDIUM_CHALLENGE_TTL_SEC || 60 * 60@@ -1238,7 +1322,7 @@ export const config: Config = {const n = Number(process.env.NOTIFICATION_DESKTOP_GRACE_SECONDS || 90)return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 90})(),- approvalRlReissuePerMin: Number(process.env.APPROVAL_RL_REISSUE_PER_MIN || 5),+ approvalRlReissuePerMin: positiveIntegerFromEnv('APPROVAL_RL_REISSUE_PER_MIN', 25),// Cleanup of stale token-bucket rows (older than 5 min — the only active// window). 5 min lines up with the longest bucket window we use.approvalRlCleanupIntervalMs: Number(process.env.APPROVAL_RL_CLEANUP_INTERVAL_MS || 5 * 60_000),diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 389f549f3..2e7c59399 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -5,6 +5,8 @@ import {createBoundedPgPoolForConnection,} from './boundedPgPool.js'import { config } from './config.js'+import { rootLogger } from './observability/logger.js'+import { applyAdminSubscriptionRateLimitNamespace } from './services/adminSubscriptionRateLimitMigration.js'import { applyCodexCatalogModelsSchema } from './services/codexSubscriptionCatalog.js'import {applyCodexChatgptAccountIdSchema,@@ -6324,6 +6326,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [version: '0124_entity_change_definer_search_path',apply: applyEntityChangeDefinerSearchPathSchema,},+ {+ version: '0125_admin_subscription_rate_limit_namespace',+ apply: applyAdminSubscriptionRateLimitNamespace,+ },]async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {@@ -6675,7 +6681,7 @@ export async function initDb(db: DbConnector = pool): Promise<void> {try {await client.query('ROLLBACK')} catch (rollbackError) {- console.warn('[ControlAPI] initDb rollback failed:', rollbackError)+ rootLogger.warn({ err: rollbackError }, 'Control API migration rollback failed')}}throw error@@ -6684,7 +6690,7 @@ export async function initDb(db: DbConnector = pool): Promise<void> {try {await client.query(`SELECT pg_advisory_unlock(${INIT_DB_LOCK_KEY_SQL})`)} catch (unlockError) {- console.warn('[ControlAPI] initDb advisory unlock failed:', unlockError)+ rootLogger.warn({ err: unlockError }, 'Control API migration advisory unlock failed')}}client.release()diff --git a/control-api/src/middleware/calendarMinuteRateLimitStore.ts b/control-api/src/middleware/calendarMinuteRateLimitStore.tsnew file mode 100644index 000000000..b4253b762--- /dev/null+++ b/control-api/src/middleware/calendarMinuteRateLimitStore.ts@@ -0,0 +1,38 @@+import { type ClientRateLimitInfo, MemoryStore } from 'express-rate-limit'++const MINUTE_MS = 60_000++/**+ * Verified-principal edge counters use the same calendar minute as the PG+ * ledger. Server-generated source-IP keys retain MemoryStore's first-hit+ * window so the independent anonymous safeguard keeps its existing contract.+ */+export class CalendarMinuteRateLimitStore extends MemoryStore {+ private scopedKey(key: string, nowMs: number): string {+ return key.includes(':ip:') ? key : `${Math.floor(nowMs / MINUTE_MS)}:${key}`+ }++ override async increment(key: string): Promise<ClientRateLimitInfo> {+ const nowMs = Date.now()+ const result = await super.increment(this.scopedKey(key, nowMs))+ return key.includes(':ip:')+ ? result+ : { ...result, resetTime: new Date((Math.floor(nowMs / MINUTE_MS) + 1) * MINUTE_MS) }+ }++ override async get(key: string): Promise<ClientRateLimitInfo | undefined> {+ const nowMs = Date.now()+ const result = await super.get(this.scopedKey(key, nowMs))+ return !result || key.includes(':ip:')+ ? result+ : { ...result, resetTime: new Date((Math.floor(nowMs / MINUTE_MS) + 1) * MINUTE_MS) }+ }++ override async decrement(key: string): Promise<void> {+ await super.decrement(this.scopedKey(key, Date.now()))+ }++ override async resetKey(key: string): Promise<void> {+ await super.resetKey(this.scopedKey(key, Date.now()))+ }+}diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.tsindex 57bd92721..ce9aa0f74 100644--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts@@ -1,6 +1,14 @@import type { Request, RequestHandler } from 'express'+import { ipKeyGenerator, rateLimit } from 'express-rate-limit'import { config } from '../config.js'-import { mcpHostRateLimitBucketKey } from '../utils/auth/mcpHostJwtToken.js'+import { verifyInternalControlJwt } from '../utils/auth/internalControlToken.js'+import {+ mcpHostRateLimitBucketKey,+ mcpHostVerifiedRateLimitPrincipal,+ verifyMcpHostAccessJwt,+} from '../utils/auth/mcpHostJwtToken.js'+import { extractBearerToken } from '../utils/extractBearerToken.js'+import { CalendarMinuteRateLimitStore } from './calendarMinuteRateLimitStore.js'import type { UiAuthedRequest } from './controlUIAuth.js'import { rateLimitMiddleware } from './rateLimitMiddleware.js'@@ -18,6 +26,102 @@ export function pluginWorkloadSdkCredentialBucketKey(req: Request): string | nulreturn mcpHostRateLimitBucketKey('plugin_workload_sdk_credential', req.mcpHostJwt)}+const preauthPrincipalCache = new WeakMap<Request, string | null>()++function verifiedPluginSdkPreauthPrincipal(req: Request): string | null {+ const cached = preauthPrincipalCache.get(req)+ if (cached !== undefined) return cached+ const principal =+ mcpHostVerifiedRateLimitPrincipal(verifyMcpHostAccessJwt(extractBearerToken(req))) || null+ preauthPrincipalCache.set(req, principal)+ return principal+}++/**+ * Anonymous and invalid credentials stay on the source-IP pre-auth ceiling.+ * A verified Host principal gets a separate allowance so that ceiling cannot+ * silently cap authenticated SDK traffic.+ */+export function pluginSdkPreauthAssignment(principal: string | null): {+ limit: number+ key: string | null+} {+ if (principal) {+ return {+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,+ key: `plugin_workload_sdk_preauth:${principal}`,+ }+ }+ return { limit: config.pluginSdkPreauthRlPerMin, key: null }+}++export function createPluginWorkloadSdkAnonymousPreauthRateLimit(): RequestHandler {+ return rateLimit({+ windowMs: 60_000,+ limit: config.pluginSdkPreauthRlPerMin,+ skip: req => verifiedPluginSdkPreauthPrincipal(req) !== null,+ standardHeaders: 'draft-8',+ legacyHeaders: false,+ message: { error: 'Too Many Requests', retryable: true },+ })+}++export function createPluginWorkloadSdkAuthenticatedPreauthRateLimit(): RequestHandler {+ return rateLimit({+ windowMs: 60_000,+ store: new CalendarMinuteRateLimitStore(),+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,+ skipSuccessfulRequests: false,+ skipFailedRequests: false,+ skip: req => verifiedPluginSdkPreauthPrincipal(req) === null,+ keyGenerator: req => {+ const assignment = pluginSdkPreauthAssignment(verifiedPluginSdkPreauthPrincipal(req))+ return assignment.key ?? 'plugin_workload_sdk_preauth:unauthenticated'+ },+ standardHeaders: 'draft-8',+ legacyHeaders: false,+ message: { error: 'Too Many Requests', retryable: true },+ })+}++const internalEdgePrincipalCache = new WeakMap<Request, string | null>()++function verifiedInternalSdkEdgePrincipal(req: Request): string | null {+ const cached = internalEdgePrincipalCache.get(req)+ if (cached !== undefined) return cached+ const token = extractBearerToken(req)+ const claims = token && token.length <= 4096 ? verifyInternalControlJwt(token) : null+ // Lossless UTF-16 encoding avoids the reserved source-IP delimiter and URI+ // encoder errors for any string accepted by the existing signed verifier.+ // This is edge attribution, never route authorization.+ const principal = claims+ ? `${claims.iss}:${Buffer.from(claims.sub, 'utf16le').toString('base64url')}`+ : null+ internalEdgePrincipalCache.set(req, principal)+ return principal+}++/** Protect JWT verification without combining verified internal callers behind one IP. */+export function createPluginWorkloadSdkInternalEdgeRateLimit(): RequestHandler {+ return rateLimit({+ windowMs: 60_000,+ store: new CalendarMinuteRateLimitStore(),+ limit: req =>+ verifiedInternalSdkEdgePrincipal(req) !== null ? config.pluginSdkInternalRlPerMin : 600,+ keyGenerator: req => {+ const principal = verifiedInternalSdkEdgePrincipal(req)+ return principal !== null+ ? `plugin_workload_sdk_internal_edge:${principal}`+ : `plugin_workload_sdk_internal_edge:ip:${ipKeyGenerator(req.ip || 'unknown')}`+ },+ skipSuccessfulRequests: false,+ skipFailedRequests: false,+ standardHeaders: 'draft-8',+ legacyHeaders: false,+ message: { error: 'Too Many Requests', retryable: true },+ })+}+/*** Distributed recipe-scoped guard for all authenticated SDK gateway routes.* The pre-auth IP limiter cannot provide caller isolation, while this PG@@ -40,7 +144,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {return rateLimitMiddleware({bucketType: 'plugin_workload_sdk_internal',- maxPerMinute: 120, // not ENV: not part of the plugin abuse surface (issue #348)+ maxPerMinute: config.pluginSdkInternalRlPerMin,getBucketKey: req => {const claims = req.internalControlif (!claims) return 'plugin_workload_sdk_internal:unauthenticated'@@ -59,7 +163,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {return rateLimitMiddleware({bucketType: 'plugin_workload_sdk_admin',- maxPerMinute: 120, // not ENV: not part of the plugin abuse surface (issue #348)+ maxPerMinute: config.pluginSdkAdminRlPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.tsindex fca6bc1c3..3b456fd3f 100644--- a/control-api/src/middleware/rateLimitMiddleware.ts+++ b/control-api/src/middleware/rateLimitMiddleware.ts@@ -33,6 +33,25 @@ export type RateLimitEnforcerOptions = {*/export type RateLimitEnforcer = (req: Request, res: Response, key: string) => Promise<boolean>+/** The last enforcing counter owns every advertised quota/header family. */+function writeRateLimitHeaders(+ res: Response,+ limit: number,+ remaining: number,+ resetMs: number+): void {+ const resetSeconds = Math.max(1, Math.ceil((resetMs - Date.now()) / 1000))+ res.setHeader('X-RateLimit-Limit', String(limit))+ res.setHeader('X-RateLimit-Remaining', String(remaining))+ res.setHeader('X-RateLimit-Reset', String(Math.floor(resetMs / 1000)))+ res.setHeader('RateLimit', `limit=${limit}, remaining=${remaining}, reset=${resetSeconds}`)+ res.setHeader('RateLimit-Policy', `${limit};w=60`)+ // Keep existing draft-6 clients consistent with the ledger as well.+ res.setHeader('RateLimit-Limit', String(limit))+ res.setHeader('RateLimit-Remaining', String(remaining))+ res.setHeader('RateLimit-Reset', String(resetSeconds))+}+/*** Factory for a per-request rate limit middleware backed by a PG token bucket.*@@ -132,6 +151,11 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLimres.removeHeader('X-RateLimit-Limit')res.removeHeader('X-RateLimit-Remaining')res.removeHeader('X-RateLimit-Reset')+ res.removeHeader('RateLimit')+ res.removeHeader('RateLimit-Policy')+ res.removeHeader('RateLimit-Limit')+ res.removeHeader('RateLimit-Remaining')+ res.removeHeader('RateLimit-Reset')res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))res.setHeader('Cache-Control', 'no-store')res.status(503).json({@@ -148,9 +172,7 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim): void {const retryAfterSec = Math.max(1, Math.ceil((denial.resetMs - Date.now()) / 1000))res.setHeader('Retry-After', String(retryAfterSec))- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', '0')- res.setHeader('X-RateLimit-Reset', String(Math.floor(denial.resetMs / 1000)))+ writeRateLimitHeaders(res, opts.maxPerMinute, 0, denial.resetMs)if (req.log) {req.log.warn({@@ -167,7 +189,12 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim'rate limit exceeded')}- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })+ res.status(429).json({+ error: 'Too Many Requests',+ code: 'rate_limited',+ message: `This request limit has been reached. Try again in ${retryAfterSec} seconds.`,+ retryAfterSeconds: retryAfterSec,+ })}return async function enforce(req: Request, res: Response, rawKey: string): Promise<boolean> {@@ -193,9 +220,12 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLimreturn false}rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))- res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))+ writeRateLimitHeaders(+ res,+ opts.maxPerMinute,+ opts.maxPerMinute - decision.totalHits,+ decision.resetMs+ )return true}if (!result.allowed) {@@ -205,9 +235,7 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim}rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))- res.setHeader('X-RateLimit-Remaining', String(result.remaining))- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))+ writeRateLimitHeaders(res, opts.maxPerMinute, result.remaining, result.resetMs)return true}}diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.tsindex 06e699ac7..0468e3dbe 100644--- a/control-api/src/routes/admin/codexSubscription.ts+++ b/control-api/src/routes/admin/codexSubscription.ts@@ -83,8 +83,8 @@ import {} from '../../services/llmAllowedModelsConfigMap.js'import { K8sConflictError } from '../../services/resourceService.js'import {- adminCodexReadRateLimits,- adminCodexWriteRateLimits,+ adminSubscriptionReadRateLimits,+ adminSubscriptionWriteRateLimits,} from '../workflows/shared/rateLimit.js'import { createHostValidationDeps, validateHostSpec } from './hostSpecValidation.js'import {@@ -819,63 +819,81 @@ export function createAdminCodexSubscriptionRouter(router.get(`${BASE}/connection`,rejectGrokUnkeyed,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),getConnectionHandler)router.post(`${BASE}/browser/start`,rejectGrokUnkeyed,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),browserStartHandler)router.post(`${BASE}/device/start`,rejectGrokUnkeyed,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),deviceStartHandler)router.get(`${BASE}/device/poll`,rejectGrokUnkeyed,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),devicePollHandler)- router.post(`${BASE}/refresh`, rejectGrokUnkeyed, ...adminCodexWriteRateLimits(), refreshHandler)+ router.post(+ `${BASE}/refresh`,+ rejectGrokUnkeyed,+ ...adminSubscriptionWriteRateLimits(),+ refreshHandler+ )router.post(`${BASE}/catalog/sync`,rejectGrokUnkeyed,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),catalogSyncHandler)- router.post(`${BASE}/revoke`, rejectGrokUnkeyed, ...adminCodexWriteRateLimits(), revokeHandler)- router.get(`${BASE}/connections/:key`, ...adminCodexReadRateLimits(), getConnectionHandler)+ router.post(+ `${BASE}/revoke`,+ rejectGrokUnkeyed,+ ...adminSubscriptionWriteRateLimits(),+ revokeHandler+ )+ router.get(`${BASE}/connections/:key`, ...adminSubscriptionReadRateLimits(), getConnectionHandler)router.post(`${BASE}/connections/:key/browser/start`,rejectGrokUnkeyed,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),browserStartHandler)router.post(`${BASE}/connections/:key/device/start`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),deviceStartHandler)router.get(`${BASE}/connections/:key/device/poll`,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),devicePollHandler)- router.post(`${BASE}/connections/:key/refresh`, ...adminCodexWriteRateLimits(), refreshHandler)+ router.post(+ `${BASE}/connections/:key/refresh`,+ ...adminSubscriptionWriteRateLimits(),+ refreshHandler+ )router.post(`${BASE}/connections/:key/catalog/sync`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),catalogSyncHandler)- router.post(`${BASE}/connections/:key/revoke`, ...adminCodexWriteRateLimits(), revokeHandler)+ router.post(+ `${BASE}/connections/:key/revoke`,+ ...adminSubscriptionWriteRateLimits(),+ revokeHandler+ )router.get(`${BASE}/connections`,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),asyncHandler(async (req, res) => {if (isGrokReq(req)) {if (!config.grokSubscriptionEnabled) {@@ -923,7 +941,7 @@ export function createAdminCodexSubscriptionRouter(router.post(`${BASE}/connections`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),asyncHandler(async (req, res) => {if (isGrokReq(req)) {if (!config.grokSubscriptionEnabled) {@@ -1002,7 +1020,7 @@ export function createAdminCodexSubscriptionRouter(router.get(`${BASE}/connections/:key/models`,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),asyncHandler(async (req, res) => {if (isGrokReq(req)) {if (!config.grokSubscriptionEnabled) {@@ -1036,7 +1054,7 @@ export function createAdminCodexSubscriptionRouter(router.patch(`${BASE}/connections/:key`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),asyncHandler(async (req, res) => {const grok = isGrokReq(req)if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {@@ -1101,7 +1119,7 @@ export function createAdminCodexSubscriptionRouter(router.patch(`${BASE}/connections/:key/models/:model`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),asyncHandler(async (req, res) => {const grok = isGrokReq(req)if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {@@ -1145,7 +1163,7 @@ export function createAdminCodexSubscriptionRouter(router.get(`${BASE}/assignable-hosts`,- ...adminCodexReadRateLimits(),+ ...adminSubscriptionReadRateLimits(),asyncHandler(async (req, res) => {if (isGrokReq(req) ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {res.status(404).json({ error: 'disabled' })@@ -1260,12 +1278,12 @@ export function createAdminCodexSubscriptionRouter(router.post(`${BASE}/connections/:key/hosts/:hostRef/unbind`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),bindUnbindHandler('unbind'))router.post(`${BASE}/connections/:key/hosts/:hostRef/bind`,- ...adminCodexWriteRateLimits(),+ ...adminSubscriptionWriteRateLimits(),bindUnbindHandler('bind'))diff --git a/control-api/src/routes/admin/index.ts b/control-api/src/routes/admin/index.tsindex 400960fc7..f14387666 100644--- a/control-api/src/routes/admin/index.ts+++ b/control-api/src/routes/admin/index.ts@@ -23,6 +23,7 @@ import { createAdminRemoteMcpRouter } from './remoteMcp.js'import { createAdminResourcesRouter } from './resources.js'import { createAdminSecretsRouter } from './secrets.js'import { createAdminSharedFilesystemsRouter } from './sharedFilesystems.js'+import { createAdminSubscriptionCapabilitiesRouter } from './subscriptionCapabilities.js'import { createAdminTeamsRouter } from './teams.js'import { createAdminTracingRouter } from './tracing/index.js'import { createAdminUsageRouter } from './usage.js'@@ -30,6 +31,7 @@ import { createAdminUsersRouter } from './users.js'export function createAdminRouter(gateway: K8sGateway): Router {const router = Router()+ router.use(createAdminSubscriptionCapabilitiesRouter())router.use(createAdminHostsOverviewRouter(gateway))router.use(createAdminControlAdminsRouter())router.use(createAdminHostArtifactsRouter(gateway))diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex 119630ad0..8ec9e9f5f 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -4922,7 +4922,7 @@ export function createAdminRegistryRouter(// ── Org-scoped publish API keys (efrk_) — owner-gated, proxied to the registry ──const keysRateLimit = rateLimitMiddleware({bucketType: 'registry_org_keys',- maxPerMinute: 30,+ maxPerMinute: config.adminRegistryKeysPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orgkeys:${sub}` : null@@ -5058,7 +5058,7 @@ export function createAdminRegistryRouter(// is audit-only (req.adminAuth.sub); it never lands in granted_by_user_id.const grantsRateLimit = rateLimitMiddleware({bucketType: 'registry_org_grants',- maxPerMinute: 30,+ maxPerMinute: config.adminRegistryGrantsPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `orggrants:${sub}` : nulldiff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.tsindex aeef9f68a..720ea1c03 100644--- a/control-api/src/routes/admin/registryConnect.ts+++ b/control-api/src/routes/admin/registryConnect.ts@@ -244,7 +244,7 @@ export function createRegistryConnectRouter(): Router {// mount, connect panel mount, refresh button).rateLimitMiddleware({bucketType: 'registry_connect_status',- maxPerMinute: 30,+ maxPerMinute: config.adminRegistryConnectStatusPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_status:${sub}` : null@@ -350,13 +350,13 @@ export function createRegistryConnectRouter(): Router {// Per-admin token bucket — registration is a rare, deliberate,// once-in-a-deployment-lifetime action, and each call consumes one of// the shared registry's ~5/day per-IP registration attempts (and can- // create a deployment row there). 3/min still allows a legitimate retry+ // create a deployment row there). 15/min still allows a legitimate retry// after a transient failure (e.g. a typo'd org name) while capping how// much of that daily budget a looped admin can burn through in a single// minute.rateLimitMiddleware({bucketType: 'registry_connect_request',- maxPerMinute: 3,+ maxPerMinute: config.adminRegistryConnectRequestPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_request:${sub}` : null@@ -612,12 +612,12 @@ export function createRegistryConnectRouter(): Router {// Per-admin token bucket — every call here rotates the deployment's// one-time claim token at the shared registry (/:id/claim-token), which// has no rate limiter of its own and emits a- // deployment.claim_token_reissued audit event on each rotate. 10/min is+ // deployment.claim_token_reissued audit event on each rotate. 50/min is// generous for a human-pressed recovery button while bounding registry// rotate/audit spam from a looped admin.rateLimitMiddleware({bucketType: 'registry_connect_recover',- maxPerMinute: 10,+ maxPerMinute: config.adminRegistryConnectRecoveryPerMin,getBucketKey: req => {const sub = (req as UiAuthedRequest).adminAuth?.subreturn sub ? `registry_connect_recover:${sub}` : nulldiff --git a/control-api/src/routes/admin/secrets.ts b/control-api/src/routes/admin/secrets.tsindex da68bc215..7ce1bcecf 100644--- a/control-api/src/routes/admin/secrets.ts+++ b/control-api/src/routes/admin/secrets.ts@@ -58,7 +58,7 @@ const defaultMcpSecretRollbackPermitStore: McpSecretRollbackPermitStore = {const mcpSecretDeleteEdgeRateLimit = rateLimit({windowMs: 60_000,- limit: 60,+ limit: config.adminConnectorDeleteEdgePerMin,standardHeaders: 'draft-7',legacyHeaders: false,// Keyed on `sub`, not `jti`: `signAdminToken` mints a fresh `jti` on every@@ -76,7 +76,7 @@ const mcpSecretDeleteEdgeRateLimit = rateLimit({const mcpSecretDeleteRateLimit = rateLimitMiddleware({bucketType: 'admin_mcp_secret_delete',- maxPerMinute: 30,+ maxPerMinute: config.adminConnectorDeletePerMin,// Same reasoning as mcpSecretDeleteEdgeRateLimit: the stable principal, so// re-authenticating does not hand the caller a fresh delete budget.getBucketKey: req => {@@ -88,7 +88,7 @@ const mcpSecretDeleteRateLimit = rateLimitMiddleware({// union has no fail-open member, so both modes still enforce.//// 'process-memory', not 'closed': this is an admin-authenticated surface that- // already sits behind mcpSecretDeleteEdgeRateLimit (in-process, 60/min), which+ // already sits behind the in-process edge limiter, which// caps it whether or not Postgres can count. 'closed' answers 503, which would// deny an operator the ability to delete a Secret precisely during a Postgres// outage — when removing a compromised Secret matters most. The two 'closed'diff --git a/control-api/src/routes/admin/subscriptionCapabilities.ts b/control-api/src/routes/admin/subscriptionCapabilities.tsnew file mode 100644index 000000000..03767e55a--- /dev/null+++ b/control-api/src/routes/admin/subscriptionCapabilities.ts@@ -0,0 +1,22 @@+import { Router } from 'express'+import { config } from '../../config.js'+import { adminSubscriptionReadRateLimits } from '../workflows/shared/rateLimit.js'++/** Feature availability only: this never grants account/model execution authority. */+export function createAdminSubscriptionCapabilitiesRouter(): Router {+ const router = Router()+ router.get(+ '/admin/llm/providers/capabilities',+ ...adminSubscriptionReadRateLimits(),+ (_req, res) => {+ res.setHeader('Cache-Control', 'private, no-store')+ res.json({+ providers: {+ 'codex-subscription': { enabled: config.codexSubscriptionEnabled },+ 'grok-subscription': { enabled: config.grokSubscriptionEnabled },+ },+ })+ }+ )+ return router+}diff --git a/control-api/src/routes/auth/codexSubscription.ts b/control-api/src/routes/auth/codexSubscription.tsindex 2f974bca5..75717a13d 100644--- a/control-api/src/routes/auth/codexSubscription.ts+++ b/control-api/src/routes/auth/codexSubscription.ts@@ -22,7 +22,7 @@ import {resolveCodexCallbackControlUiBaseUrl,} from '../../services/codexSubscriptionRedirectUri.js'import { publishAllowedModelsConfigMapAfterGrantChange } from '../../services/llmAllowedModelsConfigMap.js'-import { codexOAuthCallbackRateLimits } from '../workflows/shared/rateLimit.js'+import { subscriptionOAuthCallbackRateLimits } from '../workflows/shared/rateLimit.js'const log = rootLogger.child({ module: 'auth-codex-subscription' })@@ -65,7 +65,7 @@ export function createAuthCodexSubscriptionRouter(router.get('/auth/codex-subscription/callback',- ...codexOAuthCallbackRateLimits(),+ ...subscriptionOAuthCallbackRateLimits(),asyncHandler(async (req, res) => {const code = typeof req.query.code === 'string' ? req.query.code : ''const state = typeof req.query.state === 'string' ? req.query.state : ''@@ -88,7 +88,10 @@ export function createAuthCodexSubscriptionRouter(redirectToCodexSurface(res, 'connected')} catch (err) {if (err instanceof CodexSubscriptionOAuthError) {- log.warn({ event: 'codex_oauth_callback_denied', code: err.code }, 'callback denied')+ log.warn(+ { event: 'subscription_oauth_callback_denied', code: err.code },+ 'callback denied'+ )if (err.code === 'disabled') {res.status(404).json({ error: err.code })returndiff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.tsindex 0faa9c705..343d5d073 100644--- a/control-api/src/routes/gfs/grants.ts+++ b/control-api/src/routes/gfs/grants.ts@@ -1,5 +1,6 @@import type { Request, Router } from 'express'import { createHash } from 'node:crypto'+import { config } from '../../config.js'import { type DbClient, pool, withTransaction } from '../../db.js'import type { ExternalGfsAuthority } from '../../gfs/externalAuthority.js'import { isValidHostSubjectId } from '../../gfs/hostSubject.js'@@ -891,7 +892,7 @@ export function registerGfsGrantRoutes(router: Router): void {// first, so adminAuth.sub is always present by the time the limiter keys.const grantsRateLimit = rateLimitMiddleware({bucketType: 'gfs_grants',- maxPerMinute: 30,+ maxPerMinute: config.adminGfsGrantsPerMin,getBucketKey: req => {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants:${sub}` : nulldiff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.tsindex 16daaf772..38187cc20 100644--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts@@ -1,4 +1,5 @@import type { Request, Response, Router } from 'express'+import { config } from '../../config.js'import { pool } from '../../db.js'import { asyncHandler } from '../../http/asyncHandler.js'import { requireAuthForControlUI } from '../../middleware/controlUIAuth.js'@@ -64,7 +65,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {// consuming the grant-mutation budget while still bounding operator polling.const legacyGrantReportRateLimit = rateLimitMiddleware({bucketType: 'gfs_grants_legacy_report',- maxPerMinute: 30,+ maxPerMinute: config.adminGfsLegacyGrantReportPerMin,getBucketKey: req => {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsgrants-legacy:${sub}` : nulldiff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.tsindex 0bdd0c325..73e45d244 100644--- a/control-api/src/routes/gfs/shares.ts+++ b/control-api/src/routes/gfs/shares.ts@@ -1,4 +1,5 @@import type { Request, Response, Router } from 'express'+import { config } from '../../config.js'import { pool, withTransaction } from '../../db.js'import { asyncHandler } from '../../http/asyncHandler.js'import { requireAuthForControlUI } from '../../middleware/controlUIAuth.js'@@ -40,7 +41,7 @@ export function registerGfsShareRoutes(router: Router): void {// Per-admin token bucket, same shape as the folder-grant routes above.const sharesRateLimit = rateLimitMiddleware({bucketType: 'gfs_shares',- maxPerMinute: 30,+ maxPerMinute: config.adminGfsSharesPerMin,getBucketKey: req => {const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.subreturn sub ? `gfsshares:${sub}` : nulldiff --git a/control-api/src/routes/internal/pluginWorkloadSdk.ts b/control-api/src/routes/internal/pluginWorkloadSdk.tsindex 0cee68cb6..4d5edee94 100644--- a/control-api/src/routes/internal/pluginWorkloadSdk.ts+++ b/control-api/src/routes/internal/pluginWorkloadSdk.ts@@ -1,8 +1,10 @@import { Router } from 'express'-import { rateLimit } from 'express-rate-limit'import { asyncHandler } from '../../http/asyncHandler.js'import { requireInternalControlJwt } from '../../middleware/internalControlJwt.js'-import { createPluginWorkloadSdkInternalRateLimit } from '../../middleware/pluginWorkloadSdkRateLimits.js'+import {+ createPluginWorkloadSdkInternalEdgeRateLimit,+ createPluginWorkloadSdkInternalRateLimit,+} from '../../middleware/pluginWorkloadSdkRateLimits.js'import {type PluginWorkloadSdkRevocationActor,finalizePluginWorkloadSdkRevocation,@@ -57,13 +59,7 @@ export function createInternalPluginWorkloadSdkRouter(): Router {// the prefix so every current/future internal SDK route inherits the guard.router.use('/internal/plugin-workload-sdk',- rateLimit({- windowMs: 60_000,- limit: 600,- standardHeaders: 'draft-8',- legacyHeaders: false,- message: { error: 'Too Many Requests', retryable: true },- }),+ createPluginWorkloadSdkInternalEdgeRateLimit(),requireInternalControlJwt,createPluginWorkloadSdkInternalRateLimit())diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.tsindex 931f43628..d85177a35 100644--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts@@ -1,11 +1,12 @@import { type NextFunction, type Request, type Response, Router } from 'express'-import { rateLimit } from 'express-rate-limit'import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'import { config } from '../../config.js'import { pool } from '../../db.js'import { asyncHandler } from '../../http/asyncHandler.js'import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'import {+ createPluginWorkloadSdkAnonymousPreauthRateLimit,+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit,createPluginWorkloadSdkRequestRateLimit,pluginWorkloadSdkCredentialBucketKey,} from '../../middleware/pluginWorkloadSdkRateLimits.js'@@ -80,7 +81,7 @@ export const PLUGIN_WORKLOAD_SDK_PROMPT_BRIDGE_CONTRACT_VERSION = 2// grant rate/quota remains the product-level invocation limit.const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({bucketType: 'plugin_workload_sdk_credential',- maxPerMinute: 120,+ maxPerMinute: config.pluginSdkCredentialRlPerMin,getBucketKey: pluginWorkloadSdkCredentialBucketKey,onBackendUnavailable: 'process-memory',})@@ -626,15 +627,11 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {// recipe-scoped PG limiter (the outer app bounds JSON bodies).router.use('/mcp-host/plugin-workload-sdk',- rateLimit({- windowMs: 60_000,- // ENV-tunable pre-auth flood guard (issue #348):- // CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN.- limit: config.pluginSdkPreauthRlPerMin,- standardHeaders: 'draft-8',- legacyHeaders: false,- message: { error: 'Too Many Requests', retryable: true },- }),+ // Invalid or missing credentials stay on the anonymous source-IP gate.+ // Verified Host principals use a separate pre-auth allowance, then the+ // recipe-scoped request bucket after authentication.+ createPluginWorkloadSdkAnonymousPreauthRateLimit(),+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit(),requireMcpHostJwt,requirePluginWorkloadSdkScope,createPluginWorkloadSdkRequestRateLimit()diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.tsindex f1f2e5edf..d679444a7 100644--- a/control-api/src/routes/workflows/shared/rateLimit.ts+++ b/control-api/src/routes/workflows/shared/rateLimit.ts@@ -1,8 +1,13 @@-import type { Request, Response } from 'express'-import { ipKeyGenerator, rateLimit } from 'express-rate-limit'+import type { Request, RequestHandler, Response } from 'express'+import { type RateLimitInfo, ipKeyGenerator, rateLimit } from 'express-rate-limit'import { createHash } from 'node:crypto'import { config } from '../../../config.js'-import { rateLimitMiddleware } from '../../../middleware/rateLimitMiddleware.js'+import { CalendarMinuteRateLimitStore } from '../../../middleware/calendarMinuteRateLimitStore.js'+import {+ type RateLimitEnforcerOptions,+ rateLimitMiddleware,+} from '../../../middleware/rateLimitMiddleware.js'+import { rateLimitHitsTotal } from '../../../observability/metrics.js'import { verifyAdminToken } from '../../../utils/auth/adminAuthToken.js'import { verifyExternalSessionToken } from '../../../utils/auth/externalSessionAuthToken.js'import {@@ -12,12 +17,31 @@ import {import { CONTROL_UI_ADMIN_SESSION_COOKIE, readCookie } from '../../../utils/auth/sessionCookies.js'import { extractBearerToken } from '../../../utils/extractBearerToken.js'-const WORKFLOW_GRANT_READ_PER_MINUTE = 60-const WORKFLOW_GRANT_WRITE_PER_MINUTE = 20-const WORKFLOW_ADMIN_READ_PER_MINUTE = 60-const ADMIN_OUTPUTS_READ_PER_MINUTE = 30+// Invalid credentials still use the original source-IP abuse budgets. Raised+// administrative capacity applies only after the existing signature check.+const UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE = 60+const UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE = 20+const UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE = 60+const UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE = 30const WORKFLOW_TRIGGER_PER_MINUTE = 10-const LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE = 60+const UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE = 30+const UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE = 20+const SUBSCRIPTION_CALLBACK_IP_PER_MINUTE = 20++function hasVerifiedAdminCredential(req: Request): boolean {+ return verifiedAdminRateLimitIdentity(adminWorkflowRateLimitCredential(req)) !== null+}++/** Reuse the existing enforcer without raising a distinct unauthenticated gate. */+function withUnverifiedIpBudget(+ options: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null },+ unverifiedMaxPerMinute: number,+ isVerified: (req: Request) => boolean+): RequestHandler {+ const verified = rateLimitMiddleware(options)+ const unverified = rateLimitMiddleware({ ...options, maxPerMinute: unverifiedMaxPerMinute })+ return (req, res, next) => (isVerified(req) ? verified : unverified)(req, res, next)+}/*** Credential surface matched by requireAdminWorkflowCaller: bearer for automation,@@ -37,9 +61,12 @@ export function verifiedAdminRateLimitSubject(credential: string | null): stringreturn claims?.sub || null}-function hashedAdminWorkflowCredentialBucket(prefix: string) {+function hashedAdminWorkflowCredentialBucket(+ prefix: string,+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential+) {return (req: Request): string | null => {- const credential = adminWorkflowRateLimitCredential(req)+ const credential = getCredential(req)const sessionIdentity = verifiedAdminRateLimitIdentity(credential)if (sessionIdentity) {const hash = createHash('sha256').update(sessionIdentity).digest('hex').slice(0, 32)@@ -65,8 +92,12 @@ export function shouldSkipWorkflowGrantEdgeRateLimit(req: Request): boolean {* - Verified admin cookie/bearer → per-session bucket (isolates live sessions)* - Unverified credential → IP bucket (rotation cannot mint fresh identities)*/-export function workflowGrantEdgeRateLimitKey(prefix: string, req: Request): string {- const sessionIdentity = verifiedAdminRateLimitIdentity(adminWorkflowRateLimitCredential(req))+export function workflowGrantEdgeRateLimitKey(+ prefix: string,+ req: Request,+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential+): string {+ const sessionIdentity = verifiedAdminRateLimitIdentity(getCredential(req))if (sessionIdentity) {const hash = createHash('sha256').update(sessionIdentity).digest('hex').slice(0, 32)return `${prefix}:sub:${hash}`@@ -74,11 +105,11 @@ export function workflowGrantEdgeRateLimitKey(prefix: string, req: Request): strreturn `${prefix}:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`}-function workflowGrantEdgeRateKey(prefix: string) {- return (req: Request): string => workflowGrantEdgeRateLimitKey(prefix, req)+function workflowGrantEdgeRateKey(prefix: string, getCredential: (req: Request) => string | null) {+ return (req: Request): string => workflowGrantEdgeRateLimitKey(prefix, req, getCredential)}-function workflowGrantEdgeRateLimitHandler(_req: Request, res: Response): void {+function edgeRateLimitDenied(bucketType: string, req: Request, res: Response): void {const raw = res.getHeader('Retry-After')const retryAfterSeconds =typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0@@ -86,18 +117,44 @@ function workflowGrantEdgeRateLimitHandler(_req: Request, res: Response): void {: typeof raw === 'string' && /^\d+$/.test(raw)? Math.max(1, Number(raw)): 60- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })+ const info = (req as Request & { rateLimit?: RateLimitInfo }).rateLimit+ rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'denied' }, 1)+ req.log?.warn(+ {+ event: 'rate_limit_denied',+ bucketType,+ count: info?.used,+ maxPerMinute: info?.limit,+ source: 'edge-memory',+ method: req.method,+ route: typeof req.route?.path === 'string' ? req.route.path : undefined,+ },+ 'rate limit exceeded'+ )+ res.status(429).json({+ error: 'Too Many Requests',+ code: 'rate_limited',+ message: `This request limit has been reached. Try again in ${retryAfterSeconds} seconds.`,+ retryAfterSeconds,+ })}-function createWorkflowEdgeRateLimit(prefix: string, limit: number) {+function createWorkflowEdgeRateLimit(+ prefix: string,+ limit: number,+ unverifiedLimit: number,+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential+) {return rateLimit({windowMs: 60_000,- limit,+ store: new CalendarMinuteRateLimitStore(),+ limit: req =>+ verifiedAdminRateLimitIdentity(getCredential(req)) !== null ? limit : unverifiedLimit,standardHeaders: 'draft-7',legacyHeaders: false,- skip: shouldSkipWorkflowGrantEdgeRateLimit,- keyGenerator: workflowGrantEdgeRateKey(prefix),- handler: workflowGrantEdgeRateLimitHandler,+ skip: req => getCredential(req) === null,+ keyGenerator: workflowGrantEdgeRateKey(prefix, getCredential),+ handler: (req, res) => edgeRateLimitDenied(prefix, req, res),})}@@ -106,11 +163,19 @@ function createWorkflowEdgeRateLimit(prefix: string, limit: number) {* Routers that own a rate-limit family instantiate the tuple once and reuse it.*/export function workflowGrantReadEdgeRateLimit() {- return createWorkflowEdgeRateLimit('workflow_grants_read_edge', WORKFLOW_GRANT_READ_PER_MINUTE)+ return createWorkflowEdgeRateLimit(+ 'workflow_grants_read_edge',+ config.adminWorkflowGrantReadPerMin,+ UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE+ )}export function workflowGrantWriteEdgeRateLimit() {- return createWorkflowEdgeRateLimit('workflow_grants_write_edge', WORKFLOW_GRANT_WRITE_PER_MINUTE)+ return createWorkflowEdgeRateLimit(+ 'workflow_grants_write_edge',+ config.adminWorkflowGrantWritePerMin,+ UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE+ )}export function workflowGrantReadRateLimits() {@@ -122,11 +187,19 @@ export function workflowGrantWriteRateLimits() {}function workflowAdminReadEdgeRateLimit() {- return createWorkflowEdgeRateLimit('workflow_admin_read_edge', WORKFLOW_ADMIN_READ_PER_MINUTE)+ return createWorkflowEdgeRateLimit(+ 'workflow_admin_read_edge',+ config.adminWorkflowReadPerMin,+ UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE+ )}function adminOutputsReadEdgeRateLimit() {- return createWorkflowEdgeRateLimit('admin_outputs_read_edge', ADMIN_OUTPUTS_READ_PER_MINUTE)+ return createWorkflowEdgeRateLimit(+ 'admin_outputs_read_edge',+ config.adminOutputsReadPerMin,+ UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE+ )}export function workflowAdminReadRateLimits() {@@ -137,65 +210,108 @@ export function adminOutputsReadRateLimits() {return [adminOutputsReadEdgeRateLimit(), adminOutputsReadRateLimit()] as const}-function adminCodexReadEdgeRateLimit() {- return createWorkflowEdgeRateLimit('admin_codex_read_edge', config.adminSubscriptionReadPerMin)+/** Subscription routes follow their cookie-only parent authentication. */+function adminSubscriptionCredential(req: Request): string | null {+ return readCookie(req, CONTROL_UI_ADMIN_SESSION_COOKIE) || null}-function adminCodexWriteEdgeRateLimit() {- return createWorkflowEdgeRateLimit('admin_codex_write_edge', config.adminSubscriptionWritePerMin)+function hasVerifiedSubscriptionCredential(req: Request): boolean {+ return verifiedAdminRateLimitIdentity(adminSubscriptionCredential(req)) !== null}-function adminCodexReadRateLimit() {- return rateLimitMiddleware({- bucketType: 'admin_codex_read',- maxPerMinute: config.adminSubscriptionReadPerMin,- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),- onBackendUnavailable: 'process-memory',- })+function adminSubscriptionReadEdgeRateLimit() {+ return createWorkflowEdgeRateLimit(+ 'admin_subscription_read_edge',+ config.adminSubscriptionReadPerMin,+ UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE,+ adminSubscriptionCredential+ )}-function adminCodexWriteRateLimit() {- return rateLimitMiddleware({- bucketType: 'admin_codex_write',- maxPerMinute: config.adminSubscriptionWritePerMin,- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),- onBackendUnavailable: 'process-memory',- })+function adminSubscriptionWriteEdgeRateLimit() {+ return createWorkflowEdgeRateLimit(+ 'admin_subscription_write_edge',+ config.adminSubscriptionWritePerMin,+ UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE,+ adminSubscriptionCredential+ )}-export function adminCodexReadRateLimits() {- return [adminCodexReadEdgeRateLimit(), adminCodexReadRateLimit()] as const+function adminSubscriptionReadRateLimit() {+ return withUnverifiedIpBudget(+ {+ bucketType: 'admin_subscription_read',+ maxPerMinute: config.adminSubscriptionReadPerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket(+ 'admin_subscription_read',+ adminSubscriptionCredential+ ),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE,+ hasVerifiedSubscriptionCredential+ )}-export function adminCodexWriteRateLimits() {- return [adminCodexWriteEdgeRateLimit(), adminCodexWriteRateLimit()] as const+function adminSubscriptionWriteRateLimit() {+ return withUnverifiedIpBudget(+ {+ bucketType: 'admin_subscription_write',+ maxPerMinute: config.adminSubscriptionWritePerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket(+ 'admin_subscription_write',+ adminSubscriptionCredential+ ),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE,+ hasVerifiedSubscriptionCredential+ )}-function codexOAuthCallbackBucketKey(req: Request): string {+export function adminSubscriptionReadRateLimits() {+ return [adminSubscriptionReadEdgeRateLimit(), adminSubscriptionReadRateLimit()] as const+}++export function adminSubscriptionWriteRateLimits() {+ return [adminSubscriptionWriteEdgeRateLimit(), adminSubscriptionWriteRateLimit()] as const+}++function subscriptionOAuthCallbackBucketKey(req: Request): string {const state = typeof req.query.state === 'string' ? req.query.state.trim() : ''if (state) {const hash = createHash('sha256').update(state).digest('hex').slice(0, 32)- return `codex_oauth_callback:state:${hash}`+ return `subscription_oauth_callback:state:${hash}`}- return `codex_oauth_callback:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`+ return `subscription_oauth_callback:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`}-export function codexOAuthCallbackRateLimits() {+export function subscriptionOAuthCallbackRateLimits() {+ const hasState = (req: Request) =>+ typeof req.query.state === 'string' && req.query.state.trim().length > 0return [rateLimit({windowMs: 60_000,- limit: config.subscriptionOAuthCallbackPerMin,+ store: new CalendarMinuteRateLimitStore(),+ limit: req =>+ hasState(req)+ ? config.subscriptionOAuthCallbackPerMin+ : SUBSCRIPTION_CALLBACK_IP_PER_MINUTE,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: codexOAuthCallbackBucketKey,- handler: workflowGrantEdgeRateLimitHandler,- }),- rateLimitMiddleware({- bucketType: 'codex_oauth_callback',- maxPerMinute: config.subscriptionOAuthCallbackPerMin,- getBucketKey: codexOAuthCallbackBucketKey,- onBackendUnavailable: 'process-memory',+ keyGenerator: subscriptionOAuthCallbackBucketKey,+ handler: (req, res) => edgeRateLimitDenied('subscription_oauth_callback_edge', req, res),}),+ withUnverifiedIpBudget(+ {+ bucketType: 'subscription_oauth_callback',+ maxPerMinute: config.subscriptionOAuthCallbackPerMin,+ getBucketKey: subscriptionOAuthCallbackBucketKey,+ onBackendUnavailable: 'process-memory',+ },+ SUBSCRIPTION_CALLBACK_IP_PER_MINUTE,+ hasState+ ),] as const}@@ -205,7 +321,7 @@ export function codexOAuthCallbackRateLimits() {* 1st-party hosts which share that sentinel and must key by `hostRefs[0]`* - Missing or unverified bearer → client IP (rotation cannot mint buckets)*/-export function mcpHostAttemptRateLimitKey(req: Request): string {+function mcpHostAttemptRateLimitPrincipal(req: Request): string | null {const attached = mcpHostVerifiedRateLimitPrincipal(req.mcpHostJwt)// Always verify. Gating on bearer truthiness is a user-controlled skip of// the security check (CodeQL js/user-controlled-bypass). Empty or forged@@ -213,27 +329,40 @@ export function mcpHostAttemptRateLimitKey(req: Request): string {const verified = mcpHostVerifiedRateLimitPrincipal(verifyMcpHostAccessJwt(extractBearerToken(req)))- const principal = attached ?? verified+ return attached ?? verified+}++export function mcpHostAttemptRateLimitKey(req: Request): string {+ const principal = mcpHostAttemptRateLimitPrincipal(req)if (principal) return `llm_provider_attempt:${principal}`return `llm_provider_attempt:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`}export function llmProviderAttemptAuthorizeRateLimits() {+ const hasPrincipal = (req: Request) => mcpHostAttemptRateLimitPrincipal(req) !== nullreturn [rateLimit({windowMs: 60_000,- limit: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,+ store: new CalendarMinuteRateLimitStore(),+ limit: req =>+ hasPrincipal(req)+ ? config.llmProviderAttemptAuthorizePerMin+ : config.llmProviderAttemptAuthorizeAnonymousIpPerMin,standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: mcpHostAttemptRateLimitKey,- handler: workflowGrantEdgeRateLimitHandler,- }),- rateLimitMiddleware({- bucketType: 'llm_provider_attempt_authorize',- maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,- getBucketKey: mcpHostAttemptRateLimitKey,- onBackendUnavailable: 'process-memory',+ handler: (req, res) => edgeRateLimitDenied('llm_provider_attempt_authorize_edge', req, res),}),+ withUnverifiedIpBudget(+ {+ bucketType: 'llm_provider_attempt_authorize',+ maxPerMinute: config.llmProviderAttemptAuthorizePerMin,+ getBucketKey: mcpHostAttemptRateLimitKey,+ onBackendUnavailable: 'process-memory',+ },+ config.llmProviderAttemptAuthorizeAnonymousIpPerMin,+ hasPrincipal+ ),] as const}@@ -249,12 +378,16 @@ function adminWorkflowTriggerRateLimitKey(req: Request): string | null {}export function adminWorkflowTriggerRateLimit() {- return rateLimitMiddleware({- bucketType: 'workflow_trigger',- maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,- getBucketKey: adminWorkflowTriggerRateLimitKey,- onBackendUnavailable: 'process-memory',- })+ return withUnverifiedIpBudget(+ {+ bucketType: 'workflow_trigger',+ maxPerMinute: config.adminWorkflowTriggerPerMin,+ getBucketKey: adminWorkflowTriggerRateLimitKey,+ onBackendUnavailable: 'process-memory',+ },+ WORKFLOW_TRIGGER_PER_MINUTE,+ hasVerifiedAdminCredential+ )}function hashedWorkflowTriggerBucket(credential: string): string {@@ -326,39 +459,55 @@ export function mcpHostWorkflowTriggerRateLimit() {}function workflowAdminReadRateLimit() {- return rateLimitMiddleware({- bucketType: 'workflow_admin_read',- maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),- onBackendUnavailable: 'process-memory',- })+ return withUnverifiedIpBudget(+ {+ bucketType: 'workflow_admin_read',+ maxPerMinute: config.adminWorkflowReadPerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE,+ hasVerifiedAdminCredential+ )}function adminOutputsReadRateLimit() {- return rateLimitMiddleware({- bucketType: 'admin_outputs_read',- maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),- onBackendUnavailable: 'process-memory',- })+ return withUnverifiedIpBudget(+ {+ bucketType: 'admin_outputs_read',+ maxPerMinute: config.adminOutputsReadPerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE,+ hasVerifiedAdminCredential+ )}export function workflowGrantReadRateLimit() {- return rateLimitMiddleware({- bucketType: 'workflow_grants_read',- maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),- onBackendUnavailable: 'process-memory',- })+ return withUnverifiedIpBudget(+ {+ bucketType: 'workflow_grants_read',+ maxPerMinute: config.adminWorkflowGrantReadPerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE,+ hasVerifiedAdminCredential+ )}export function workflowGrantWriteRateLimit() {- return rateLimitMiddleware({- bucketType: 'workflow_grants_write',- maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),- onBackendUnavailable: 'process-memory',- })+ return withUnverifiedIpBudget(+ {+ bucketType: 'workflow_grants_write',+ maxPerMinute: config.adminWorkflowGrantWritePerMin,+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),+ onBackendUnavailable: 'process-memory',+ },+ UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE,+ hasVerifiedAdminCredential+ )}/** Keep distinct signed sessions in distinct pre-auth quota buckets. */diff --git a/control-api/src/services/adminSubscriptionRateLimitMigration.ts b/control-api/src/services/adminSubscriptionRateLimitMigration.tsnew file mode 100644index 000000000..8318ebd0b--- /dev/null+++ b/control-api/src/services/adminSubscriptionRateLimitMigration.ts@@ -0,0 +1,80 @@+import type { DbClient } from '../db.js'++const NAMESPACE_RENAMES = [+ ['admin_codex_read:', 'admin_subscription_read:'],+ ['admin_codex_write:', 'admin_subscription_write:'],+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],+] as const++/** Existing opaque long-key digests cannot be backfilled from their hashes. */+export function legacyAdministrativeRateLimitHashInput(key: string): string {+ for (const [legacy, current] of NAMESPACE_RENAMES) {+ if (key.startsWith(current)) return legacy + key.slice(current.length)+ }+ return key+}++/**+ * Preserve active quota accounting across the provider-neutral rename.+ * initDb holds a transaction and migration advisory lock. The table lock+ * serializes the backfill with increments; the invoker trigger also routes+ * older binaries to the same rows during rolling updates and rollback.+ * Legacy provider spellings are intentionally confined to this compatibility+ * boundary. Retire the trigger only after older writers/rollback are retired.+ */+export async function applyAdminSubscriptionRateLimitNamespace(db: DbClient): Promise<void> {+ const cases = NAMESPACE_RENAMES.map(+ ([legacy, current], index) => `+ ${index === 0 ? 'IF' : 'ELSIF'} left(input_key, ${legacy.length}) = '${legacy}' THEN+ canonical_key := '${current}' || substring(input_key FROM ${legacy.length + 1});+ ELSIF left(input_key, ${current.length}) = '${current}' THEN+ legacy_input := '${legacy}' || substring(input_key FROM ${current.length + 1});+ `+ ).join('\n')+ await db.query(`+ LOCK TABLE rate_limit_buckets IN ACCESS EXCLUSIVE MODE;++ CREATE OR REPLACE FUNCTION public.canonical_admin_subscription_bucket_key(input_key TEXT)+ RETURNS TEXT LANGUAGE plpgsql IMMUTABLE STRICT SECURITY INVOKER+ SET search_path = pg_catalog AS $$+ DECLARE+ canonical_key TEXT := input_key;+ legacy_input TEXT := input_key;+ BEGIN+ ${cases}+ ELSE+ RETURN input_key;+ END IF;+ IF octet_length(canonical_key) > 512 THEN+ RETURN 'sha256-long-key:' || encode(sha256(convert_to(legacy_input, 'UTF8')), 'hex');+ END IF;+ RETURN canonical_key;+ END;+ $$;++ INSERT INTO rate_limit_buckets (bucket_key, window_start_ms, count)+ SELECT public.canonical_admin_subscription_bucket_key(bucket_key),+ window_start_ms, SUM(count)::INTEGER+ FROM rate_limit_buckets+ WHERE public.canonical_admin_subscription_bucket_key(bucket_key) <> bucket_key+ GROUP BY public.canonical_admin_subscription_bucket_key(bucket_key), window_start_ms+ ON CONFLICT (bucket_key, window_start_ms)+ DO UPDATE SET count = rate_limit_buckets.count + EXCLUDED.count;+ DELETE FROM rate_limit_buckets+ WHERE public.canonical_admin_subscription_bucket_key(bucket_key) <> bucket_key;++ CREATE OR REPLACE FUNCTION public.normalize_admin_subscription_bucket_key()+ RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER+ SET search_path = pg_catalog AS $$+ BEGIN+ NEW.bucket_key := public.canonical_admin_subscription_bucket_key(NEW.bucket_key);+ RETURN NEW;+ END;+ $$;++ DROP TRIGGER IF EXISTS normalize_admin_subscription_bucket_key ON rate_limit_buckets;+ CREATE TRIGGER normalize_admin_subscription_bucket_key+ BEFORE INSERT OR UPDATE ON rate_limit_buckets+ FOR EACH ROW EXECUTE FUNCTION public.normalize_admin_subscription_bucket_key();+ `)+}diff --git a/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts b/control-api/src/services/pluginWorkloadSdkQuotaTracker.tsindex 2f74147d5..adc3c5df8 100644--- a/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts+++ b/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts@@ -13,7 +13,7 @@ import {// override still wins (decision §3.1) — it is a per-recipe fairness knob, NOT// the platform DoS ceiling, so it is intentionally NOT clamped to the default.// A raised override stays bounded by the grant-independent request bucket-// (CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN, default 600/min) + pre-auth+// (CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN, default 3000/min) + pre-auth// limits (pluginWorkloadSdkRateLimits.ts / plugin-workload-sdk.routes.ts),// which are the actual anti-DoS backstop.//diff --git a/control-api/src/services/rateLimitBucketKey.ts b/control-api/src/services/rateLimitBucketKey.tsindex 847a0daf5..53953e170 100644--- a/control-api/src/services/rateLimitBucketKey.ts+++ b/control-api/src/services/rateLimitBucketKey.ts@@ -1,4 +1,5 @@import { createHash } from 'node:crypto'+import { legacyAdministrativeRateLimitHashInput } from './adminSubscriptionRateLimitMigration.js'/** Longest bucket key, in UTF-8 bytes, that the rate limiters store as given. */export const MAX_RATE_LIMIT_BUCKET_KEY_BYTES = 512@@ -20,5 +21,8 @@ const LONG_KEY_PREFIX = 'sha256-long-key:'*/export function boundedBucketKey(key: string): string {if (Buffer.byteLength(key, 'utf8') <= MAX_RATE_LIMIT_BUCKET_KEY_BYTES) return key- return `${LONG_KEY_PREFIX}${createHash('sha256').update(key).digest('hex')}`+ // Preserve pre-rename digests: the database cannot recover a namespace from+ // an existing hash. Plain renamed keys are handled by the migration trigger.+ const hashInput = legacyAdministrativeRateLimitHashInput(key)+ return `${LONG_KEY_PREFIX}${createHash('sha256').update(hashInput).digest('hex')}`}diff --git a/control-api/test/app.test.ts b/control-api/test/app.test.tsindex 9cdbd82c8..22fb8d2f9 100644--- a/control-api/test/app.test.ts+++ b/control-api/test/app.test.ts@@ -17,6 +17,15 @@ function signHccInternalControl(): string {}describe('app router wiring', () => {+ it('requires the real admin session guard for subscription capability discovery', async () => {+ const app = createApp(new MockGateway('mcp-server') as never)+ await request(app).get('/api/v1/admin/llm/providers/capabilities').expect(401)+ await request(app)+ .get('/api/v1/admin/llm/providers/capabilities')+ .set('Cookie', 'control_ui_admin_session=unverified-fixture')+ .expect(401)+ })+afterEach(() => {vi.restoreAllMocks()})diff --git a/control-api/test/config.adminRateLimits.test.ts b/control-api/test/config.adminRateLimits.test.tsindex eb8a3caed..e4c1f4dba 100644--- a/control-api/test/config.adminRateLimits.test.ts+++ b/control-api/test/config.adminRateLimits.test.ts@@ -54,3 +54,78 @@ describe('administrative subscription capacity', () => {)}})++const OPERATING_BUDGETS = [+ ['CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN', 'adminWorkflowReadPerMin', 300],+ ['CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN', 'adminWorkflowGrantReadPerMin', 300],+ ['CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN', 'adminWorkflowGrantWritePerMin', 100],+ ['CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN', 'adminWorkflowTriggerPerMin', 50],+ ['CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN', 'adminOutputsReadPerMin', 150],+ ['CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN', 'llmProviderAttemptAuthorizePerMin', 300],+ [+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN',+ 'llmProviderAttemptAuthorizeAnonymousIpPerMin',+ 60,+ ],+ ['CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN', 'adminRegistryKeysPerMin', 150],+ ['CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN', 'adminRegistryGrantsPerMin', 150],+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN', 'adminRegistryConnectStatusPerMin', 150],+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN', 'adminRegistryConnectRequestPerMin', 15],+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN', 'adminRegistryConnectRecoveryPerMin', 50],+ ['CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN', 'adminConnectorDeleteEdgePerMin', 300],+ ['CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN', 'adminConnectorDeletePerMin', 150],+ ['CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN', 'adminGfsGrantsPerMin', 150],+ ['CONTROL_API_ADMIN_GFS_SHARES_PER_MIN', 'adminGfsSharesPerMin', 150],+ ['CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN', 'adminGfsLegacyGrantReportPerMin', 150],+ ['APPROVAL_RL_REFRESH_PER_MIN', 'approvalRlRefreshPerMin', 100],+ ['APPROVAL_RL_REISSUE_PER_MIN', 'approvalRlReissuePerMin', 25],+ ['CONTROL_API_OAUTH_BROKER_RL_PER_MIN', 'oauthBrokerRlPerMin', 300],+] as const++async function readOperatingBudgets(+ overrides: Partial<Record<(typeof OPERATING_BUDGETS)[number][0], string>> = {}+) {+ for (const [key] of OPERATING_BUDGETS) vi.stubEnv(key, overrides[key] ?? '')+ vi.resetModules()+ const { config } = await import('../src/config.js')+ return Object.fromEntries(OPERATING_BUDGETS.map(([, field]) => [field, config[field]])) as Record<+ (typeof OPERATING_BUDGETS)[number][1],+ number+ >+}++describe('similar authenticated administrative budgets', () => {+ it('defaults each family to at least five times its previous ceiling', async () => {+ const config = await readOperatingBudgets()+ for (const [, field, expected] of OPERATING_BUDGETS) {+ expect(config[field]).toBe(expected)+ }+ })++ it('keeps the anonymous provider-attempt allowance separate from the verified budget', async () => {+ const config = await readOperatingBudgets({+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN: '900',+ })+ expect(config.llmProviderAttemptAuthorizePerMin).toBe(900)+ expect(config.llmProviderAttemptAuthorizeAnonymousIpPerMin).toBe(60)+ })++ it('honors an explicit override without changing neighboring families', async () => {+ const config = await readOperatingBudgets({+ CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN: '7',+ CONTROL_API_ADMIN_GFS_SHARES_PER_MIN: '40',+ })+ expect(config.adminRegistryConnectRequestPerMin).toBe(7)+ expect(config.adminGfsSharesPerMin).toBe(40)+ expect(config.adminRegistryConnectRecoveryPerMin).toBe(50)+ expect(config.adminGfsGrantsPerMin).toBe(150)+ })++ for (const [key] of OPERATING_BUDGETS) {+ it(`rejects invalid ${key}`, async () => {+ await expect(readOperatingBudgets({ [key]: '0' })).rejects.toThrow(+ `${key} must be a positive integer`+ )+ })+ }+})diff --git a/control-api/test/config.approvalRateLimit.test.ts b/control-api/test/config.approvalRateLimit.test.tsindex ec1b3f3ef..d2dcb037e 100644--- a/control-api/test/config.approvalRateLimit.test.ts+++ b/control-api/test/config.approvalRateLimit.test.ts@@ -87,7 +87,7 @@ describe('control-api approval rate limit config', () => {it('defaults approval request rate limit to multi-agent channel-reader capacity', async () => {const config = await loadConfigWith({})- expect(config.approvalRlRequestPerMin).toBe(120)+ expect(config.approvalRlRequestPerMin).toBe(600)})it('accepts the approval request rate limit environment override', async () => {diff --git a/control-api/test/config.hostWakeRateLimit.test.ts b/control-api/test/config.hostWakeRateLimit.test.tsindex f46f7218b..4908cbb27 100644--- a/control-api/test/config.hostWakeRateLimit.test.ts+++ b/control-api/test/config.hostWakeRateLimit.test.ts@@ -137,10 +137,16 @@ describe('host-wake rate limit budget (cross-service derivation guard)', () => {)})- it('defaults host wake rate limit to the derived 30/min', async () => {+ it('defaults host wake rate limit to the authenticated 150/min budget', async () => {const config = await loadConfigWith({})- expect(config.hostWakeRlPerMin).toBe(30)+ expect(config.hostWakeRlPerMin).toBe(150)+ })++ it.each(['0', '-1', '1.5', 'nope'])('rejects invalid host wake budget %s', async value => {+ await expect(loadConfigWith({ CONTROL_API_HOST_WAKE_RL_PER_MIN: value })).rejects.toThrow(+ 'CONTROL_API_HOST_WAKE_RL_PER_MIN must be a positive integer'+ )})it('accepts the host wake rate limit environment override', async () => {diff --git a/control-api/test/config.pluginSdkRateLimits.test.ts b/control-api/test/config.pluginSdkRateLimits.test.tsindex 201f5b9b1..a0305b7c7 100644--- a/control-api/test/config.pluginSdkRateLimits.test.ts+++ b/control-api/test/config.pluginSdkRateLimits.test.ts@@ -11,9 +11,9 @@ import { readFileSync } from 'node:fs'* now live and this suite is GREEN. Do not weaken these assertions.** Covered:- * 1. Code defaults 150/120/600/600 (decision Q3: recalibrated from the old- * hardcoded 120/60; request-bucket and pre-auth unchanged at 600 but- * moved to ENV).+ * 1. Code defaults 750/600/6000/600, plus authenticated pre-auth 6000 and+ * credential operations 1800. The 600 pre-auth value remains the anonymous or+ * invalid-credential source-IP ceiling.* 2. Each ENV override honored.* 3. Invalid values fail loudly at import (positiveIntegerFromEnv).* 4. Empty string falls back to the code default.@@ -28,6 +28,10 @@ const RATE_LIMIT_KEYS = ['CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN','CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN','CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN',+ 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',+ 'CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN',+ 'CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN',+ 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',] as consttype RateLimitKey = (typeof RATE_LIMIT_KEYS)[number]@@ -37,23 +41,43 @@ const EXPECTED: Array<{ env: RateLimitKey; field: string; defaultValue: number }{env: 'CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN',field: 'pluginSdkNotificationsRlPerMin',- defaultValue: 150,+ defaultValue: 750,},{env: 'CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN',field: 'pluginSdkPromptBridgeRlPerMin',- defaultValue: 120,+ defaultValue: 600,},{env: 'CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN',field: 'pluginSdkRequestBucketRlPerMin',- defaultValue: 600,+ defaultValue: 6000,},{env: 'CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN',field: 'pluginSdkPreauthRlPerMin',defaultValue: 600,},+ {+ env: 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',+ field: 'pluginSdkAuthenticatedPreauthRlPerMin',+ defaultValue: 6000,+ },+ {+ env: 'CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN',+ field: 'pluginSdkAdminRlPerMin',+ defaultValue: 600,+ },+ {+ env: 'CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN',+ field: 'pluginSdkInternalRlPerMin',+ defaultValue: 600,+ },+ {+ env: 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',+ field: 'pluginSdkCredentialRlPerMin',+ defaultValue: 1800,+ },]async function loadConfigWith(overrides: Partial<Record<RateLimitKey, string>>) {@@ -97,13 +121,17 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {vi.resetModules()})- it('defaults the four platform rate limits to 150/120/600/600', async () => {+ it('defaults platform rate limits without raising the anonymous pre-auth ceiling', async () => {const config = await loadConfigWith({})- expect(config.pluginSdkNotificationsRlPerMin).toBe(150)- expect(config.pluginSdkPromptBridgeRlPerMin).toBe(120)- expect(config.pluginSdkRequestBucketRlPerMin).toBe(600)+ expect(config.pluginSdkNotificationsRlPerMin).toBe(750)+ expect(config.pluginSdkPromptBridgeRlPerMin).toBe(600)+ expect(config.pluginSdkRequestBucketRlPerMin).toBe(6000)expect(config.pluginSdkPreauthRlPerMin).toBe(600)+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)+ expect(config.pluginSdkAdminRlPerMin).toBe(600)+ expect(config.pluginSdkInternalRlPerMin).toBe(600)+ expect(config.pluginSdkCredentialRlPerMin).toBe(1800)})it.each(EXPECTED)('honors the $env override', async ({ env, field }) => {@@ -124,12 +152,18 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: '',CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: '',CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: '',+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: '',+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: '',+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: '',+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: '',})- expect(config.pluginSdkNotificationsRlPerMin).toBe(150)- expect(config.pluginSdkPromptBridgeRlPerMin).toBe(120)- expect(config.pluginSdkRequestBucketRlPerMin).toBe(600)+ expect(config.pluginSdkNotificationsRlPerMin).toBe(750)+ expect(config.pluginSdkPromptBridgeRlPerMin).toBe(600)+ expect(config.pluginSdkRequestBucketRlPerMin).toBe(6000)expect(config.pluginSdkPreauthRlPerMin).toBe(600)+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)+ expect(config.pluginSdkCredentialRlPerMin).toBe(1800)})it('registers every key at the code default in the base ConfigMap', async () => {@@ -168,7 +202,7 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {/maxInvocationsPerMinute:\s*\n\s*type: integer\s*\n\s*minimum: 1\s*\n\s*description: (.+)/,'maxInvocationsPerMinute description in charts/clerum-crds/crds/workflowrecipe.yaml')- expect(invocationsDescription).toContain('default 120')+ expect(invocationsDescription).toContain('default 600')expect(invocationsDescription).toContain('CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN')const notificationsDescription = extractOne(@@ -176,7 +210,7 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {/maxNotificationsPerMinute:\s*\n\s*type: integer\s*\n\s*minimum: 1\s*\n\s*description: (.+)/,'maxNotificationsPerMinute description in charts/clerum-crds/crds/workflowrecipe.yaml')- expect(notificationsDescription).toContain('default 150')+ expect(notificationsDescription).toContain('default 750')expect(notificationsDescription).toContain('CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN')})@@ -195,13 +229,39 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {expect(field).toBe('pluginSdkRequestBucketRlPerMin')})- it('wires the pre-auth flood-guard limit from config in the SDK routes', () => {- const source = read('../src/routes/mcp-host/plugin-workload-sdk.routes.ts')- const field = extractOne(- source,- /limit:\s*config\.(pluginSdkPreauthRlPerMin)\b/,- 'limit wired from config in src/routes/mcp-host/plugin-workload-sdk.routes.ts'- )- expect(field).toBe('pluginSdkPreauthRlPerMin')+ it('keeps anonymous pre-auth separate from the verified principal allowance', () => {+ const source = read('../src/middleware/pluginWorkloadSdkRateLimits.ts')+ expect(source).toContain('limit: config.pluginSdkPreauthRlPerMin')+ expect(source).toContain('limit: config.pluginSdkAuthenticatedPreauthRlPerMin')+ expect(source).toContain('maxPerMinute: config.pluginSdkAdminRlPerMin')+ expect(source).toContain('maxPerMinute: config.pluginSdkInternalRlPerMin')++ const routes = read('../src/routes/mcp-host/plugin-workload-sdk.routes.ts')+ const mounted = routes.slice(routes.indexOf("'/mcp-host/plugin-workload-sdk'"))+ const anonymous = mounted.indexOf('createPluginWorkloadSdkAnonymousPreauthRateLimit()')+ const authenticated = mounted.indexOf('createPluginWorkloadSdkAuthenticatedPreauthRateLimit()')+ const jwt = mounted.indexOf('requireMcpHostJwt,')+ const bucket = mounted.indexOf('createPluginWorkloadSdkRequestRateLimit()')+ expect(anonymous).toBeGreaterThan(-1)+ expect(anonymous).toBeLessThan(authenticated)+ expect(authenticated).toBeLessThan(jwt)+ expect(jwt).toBeLessThan(bucket)+ })++ it('gives a verified principal its own pre-auth allowance', async () => {+ const config = await loadConfigWith({})+ const { pluginSdkPreauthAssignment } =+ await import('../src/middleware/pluginWorkloadSdkRateLimits.js')++ expect(pluginSdkPreauthAssignment(null)).toEqual({+ limit: config.pluginSdkPreauthRlPerMin,+ key: null,+ })+ expect(pluginSdkPreauthAssignment('recipe:alpha')).toEqual({+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,+ key: 'plugin_workload_sdk_preauth:recipe:alpha',+ })+ expect(config.pluginSdkPreauthRlPerMin).toBe(600)+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)})})diff --git a/control-api/test/deploy.channelReaderControlApiBoundary.test.ts b/control-api/test/deploy.channelReaderControlApiBoundary.test.tsindex ec16300ee..ca27f2774 100644--- a/control-api/test/deploy.channelReaderControlApiBoundary.test.ts+++ b/control-api/test/deploy.channelReaderControlApiBoundary.test.ts@@ -128,7 +128,7 @@ describe('channel-reader -> control-api boundary manifests', () => {)expect(pollIntervalSeconds).toBe(2)- expect(approvalRequestsPerMinute).toBe(120)+ expect(approvalRequestsPerMinute).toBe(600)expect(approvalRequestsPerMinute).toBeGreaterThanOrEqual((60 / pollIntervalSeconds) * 4)})diff --git a/control-api/test/middleware.calendarMinuteRateLimitStore.test.ts b/control-api/test/middleware.calendarMinuteRateLimitStore.test.tsnew file mode 100644index 000000000..b0d255c62--- /dev/null+++ b/control-api/test/middleware.calendarMinuteRateLimitStore.test.ts@@ -0,0 +1,69 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import type { Options } from 'express-rate-limit'+import { CalendarMinuteRateLimitStore } from '../src/middleware/calendarMinuteRateLimitStore.js'++describe('CalendarMinuteRateLimitStore', () => {+ let store: CalendarMinuteRateLimitStore++ beforeEach(() => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))+ store = new CalendarMinuteRateLimitStore()+ store.init({ windowMs: 60_000 } as Options)+ })++ afterEach(() => {+ store.shutdown()+ vi.useRealTimers()+ })++ it('reports the calendar boundary and restores a verified principal at that boundary after an offset first hit', async () => {+ const key = 'plugin_workload_sdk_internal_edge:wrc:wrc-provisioner'+ await expect(store.increment(key)).resolves.toEqual({+ totalHits: 1,+ resetTime: new Date('2026-10-02T12:01:00.000Z'),+ })+ await store.increment(key)+ await expect(store.get(key)).resolves.toEqual({+ totalHits: 2,+ resetTime: new Date('2026-10-02T12:01:00.000Z'),+ })+ await vi.advanceTimersByTimeAsync(42_500)+ await expect(store.get(key)).resolves.toBeUndefined()+ await expect(store.increment(key)).resolves.toEqual({+ totalHits: 1,+ resetTime: new Date('2026-10-02T12:02:00.000Z'),+ })+ })++ it('retains the anonymous source-IP first-hit window across a calendar boundary', async () => {+ const key = 'plugin_workload_sdk_internal_edge:ip:198.51.100.9'+ await expect(store.increment(key)).resolves.toEqual({+ totalHits: 1,+ resetTime: new Date('2026-10-02T12:01:17.500Z'),+ })+ await vi.advanceTimersByTimeAsync(42_500)+ await expect(store.increment(key)).resolves.toEqual({+ totalHits: 2,+ resetTime: new Date('2026-10-02T12:01:17.500Z'),+ })+ await vi.advanceTimersByTimeAsync(17_500)+ await expect(store.increment(key)).resolves.toEqual({+ totalHits: 1,+ resetTime: new Date('2026-10-02T12:02:17.500Z'),+ })+ })++ it('uses the same scoped key for decrement and reset without affecting another principal', async () => {+ const first = 'plugin_workload_sdk_preauth:recipe-one'+ const second = 'plugin_workload_sdk_preauth:recipe-two'+ await store.increment(first)+ await store.increment(first)+ await store.increment(second)+ await store.decrement(first)+ await expect(store.get(first)).resolves.toMatchObject({ totalHits: 1 })+ await store.resetKey(first)+ await expect(store.get(first)).resolves.toBeUndefined()+ await expect(store.get(second)).resolves.toMatchObject({ totalHits: 1 })+ })+})diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.tsindex d096a78de..c21f17672 100644--- a/control-api/test/middleware.rateLimitMiddleware.test.ts+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts@@ -1,6 +1,6 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'-import { MemoryStore } from 'express-rate-limit'+import { MemoryStore, rateLimit } from 'express-rate-limit'import { createHash } from 'node:crypto'import request from 'supertest'import type { RateLimitCheck } from '../src/services/rateLimiterService.js'@@ -71,6 +71,66 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {hits.inc.mockReset()})+ it.each([1, LIMIT + 1])(+ 'publishes authoritative ledger headers after a fresh edge (count %i)',+ async count => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(resetMs - 30_000)+ try {+ checkAndIncrement.mockResolvedValue(available(count))+ const app = express()+ app.get(+ '/limited',+ rateLimit({+ windowMs: 60_000,+ limit: 100,+ standardHeaders: 'draft-7',+ legacyHeaders: false,+ }),+ rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'unit:key',+ onBackendUnavailable: 'closed',+ }),+ (_req, res) => res.sendStatus(204)+ )+ const response = await request(app).get('/limited')+ expect(response.status).toBe(count <= LIMIT ? 204 : 429)+ const remaining = Math.max(0, LIMIT - count)+ expect(response.headers.ratelimit).toBe(`limit=${LIMIT}, remaining=${remaining}, reset=30`)+ expect(response.headers['ratelimit-policy']).toBe(`${LIMIT};w=60`)+ expect(response.headers['x-ratelimit-limit']).toBe(String(LIMIT))+ expect(response.headers['x-ratelimit-remaining']).toBe(String(remaining))+ if (count > LIMIT) expect(response.headers['retry-after']).toBe('30')+ } finally {+ vi.useRealTimers()+ }+ }+ )++ it('removes stale edge quota metadata when the authoritative backend cannot count', async () => {+ checkAndIncrement.mockResolvedValue(unavailable())+ const app = express()+ app.get(+ '/limited',+ rateLimit({ windowMs: 60_000, limit: 100, standardHeaders: 'draft-7', legacyHeaders: false }),+ rateLimitMiddleware({+ bucketType: 'unit_bucket',+ maxPerMinute: LIMIT,+ getBucketKey: () => 'unit:key',+ onBackendUnavailable: 'closed',+ }),+ (_req, res) => res.sendStatus(204)+ )+ const response = await request(app).get('/limited')+ expect(response.status).toBe(503)+ expect(response.headers.ratelimit).toBeUndefined()+ expect(response.headers['ratelimit-policy']).toBeUndefined()+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()+ expect(response.headers['retry-after']).toBe('2')+ })+it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {checkAndIncrement.mockResolvedValueOnce(unavailable())const { app, handler, warn } = appWith('closed')@@ -145,7 +205,12 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {const retryAfterSeconds = Number(denied.headers['retry-after'])expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)expect(retryAfterSeconds).toBeLessThanOrEqual(60)- expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })+ expect(denied.body).toEqual({+ error: 'Too Many Requests',+ code: 'rate_limited',+ message: `This request limit has been reached. Try again in ${retryAfterSeconds} seconds.`,+ retryAfterSeconds,+ })expect(denied.headers['x-ratelimit-limit']).toBe(String(LIMIT))expect(denied.headers['x-ratelimit-remaining']).toBe('0')expect(Number(denied.headers['x-ratelimit-reset'])).toBeGreaterThan(Date.now() / 1000)diff --git a/control-api/test/pluginWorkloadSdkRateLimits.test.ts b/control-api/test/pluginWorkloadSdkRateLimits.test.tsindex bf9b5864f..a693e992b 100644--- a/control-api/test/pluginWorkloadSdkRateLimits.test.ts+++ b/control-api/test/pluginWorkloadSdkRateLimits.test.ts@@ -1,7 +1,11 @@-import { describe, expect, it } from 'vitest'-import type express from 'express'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'import { config } from '../src/config.js'+import { requireMcpHostJwt } from '../src/middleware/mcpHostJwtAuth.js'import {+ createPluginWorkloadSdkAnonymousPreauthRateLimit,+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit,pluginWorkloadSdkCredentialBucketKey,pluginWorkloadSdkRequestBucketKey,} from '../src/middleware/pluginWorkloadSdkRateLimits.js'@@ -64,3 +68,59 @@ describe('pluginWorkloadSdkRateLimits standalone isolation', () => {)})})++describe('authenticated SDK pre-auth calendar recovery', () => {+ const originalAuthenticated = config.pluginSdkAuthenticatedPreauthRlPerMin+ const originalAnonymous = config.pluginSdkPreauthRlPerMin++ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))+ config.pluginSdkAuthenticatedPreauthRlPerMin = 2+ config.pluginSdkPreauthRlPerMin = 2+ })++ afterEach(() => {+ config.pluginSdkAuthenticatedPreauthRlPerMin = originalAuthenticated+ config.pluginSdkPreauthRlPerMin = originalAnonymous+ vi.useRealTimers()+ })++ function app() {+ const instance = express()+ instance.use(createPluginWorkloadSdkAnonymousPreauthRateLimit())+ instance.use(createPluginWorkloadSdkAuthenticatedPreauthRateLimit())+ instance.get('/sdk-gate', requireMcpHostJwt, (_req, res) => res.json({ ok: true }))+ return instance+ }++ function signedCaller() {+ return issueMcpHostAccessJwt('sandbox-recipes', 'calendar-sdk', undefined, {+ workflowControlScopes: ['plugin-workload-sdk'],+ }).token+ }++ it('allows the verified caller at the PG calendar boundary instead of waiting until its first-hit deadline', async () => {+ const instance = app()+ const authorization = `Bearer ${signedCaller()}`+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)+ const denied = await request(instance).get('/sdk-gate').set('Authorization', authorization)+ expect(denied.status).toBe(429)+ expect(denied.headers['retry-after']).toBe('43')+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)+ })++ it('retains anonymous first-hit denial across that boundary without consuming the verified caller allowance', async () => {+ const instance = app()+ await request(instance).get('/sdk-gate').expect(401)+ await request(instance).get('/sdk-gate').expect(401)+ await request(instance).get('/sdk-gate').expect(429)+ const authorization = `Bearer ${signedCaller()}`+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))+ await request(instance).get('/sdk-gate').expect(429)+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)+ })+})diff --git a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.tsindex bbcc366de..f17478a2f 100644--- a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts+++ b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts@@ -332,7 +332,7 @@ describeRealPostgres(})it('enforces the per-minute window from the audit trail and frees after 61 seconds (backdated)', async () => {- // promptBridge at the 120/min platform default: 121 in-window rows deny.+ // promptBridge at the 600/min platform default: 601 in-window rows deny.const bridgeRecipe = `stepless-window-bridge-${randomBytes(4).toString('hex')}`await assertStepless(bridgeRecipe)await seedPromptBridgeGrant(bridgeRecipe, {})@@ -342,7 +342,7 @@ describeRealPostgres(if (!grant) throw new Error(`seeded promptBridge grant not found for ${bridgeRecipe}`)return grant})()- await seedInvocationWindowRows(bridgeRecipe, 'promptBridge', 'window-probe', 121)+ await seedInvocationWindowRows(bridgeRecipe, 'promptBridge', 'window-probe', 601)await expect(quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)@@ -350,7 +350,7 @@ describeRealPostgres(ok: false,error: 'quota_exceeded',retryable: false,- message: expect.stringContaining('120/minute'),+ message: expect.stringContaining('600/minute'),})// Backdate the whole burst out of the trailing minute (the window clock is@@ -365,7 +365,7 @@ describeRealPostgres(quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)).resolves.toEqual({ ok: true })- // clientNotifications mirror at the 150/min platform default, narrowed to+ // clientNotifications mirror at the 750/min platform default, narrowed to// the eventType (detail column).const notifRecipe = `stepless-window-notif-${randomBytes(4).toString('hex')}`await assertStepless(notifRecipe)@@ -376,7 +376,7 @@ describeRealPostgres(if (!grant) throw new Error(`seeded clientNotifications grant not found for ${notifRecipe}`)return grant})()- await seedInvocationWindowRows(notifRecipe, 'clientNotifications', 'lead.followup.due', 151)+ await seedInvocationWindowRows(notifRecipe, 'clientNotifications', 'lead.followup.due', 751)await expect(quotaTracker.checkRateLimit(NS, notifRecipe, 'clientNotifications', notifGrant, {@@ -386,7 +386,7 @@ describeRealPostgres(ok: false,error: 'quota_exceeded',retryable: false,- message: expect.stringContaining('150/minute'),+ message: expect.stringContaining('750/minute'),})await db.pool.query(diff --git a/control-api/test/routes.admin.subscriptionCapabilities.test.ts b/control-api/test/routes.admin.subscriptionCapabilities.test.tsnew file mode 100644index 000000000..418ecc3e6--- /dev/null+++ b/control-api/test/routes.admin.subscriptionCapabilities.test.ts@@ -0,0 +1,101 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import { config } from '../src/config.js'+import { requireAuthForControlUI } from '../src/middleware/controlUIAuth.js'+import { createAdminSubscriptionCapabilitiesRouter } from '../src/routes/admin/subscriptionCapabilities.js'+import { signAdminToken, verifyAdminToken } from '../src/utils/auth/adminAuthToken.js'++const authenticate = vi.hoisted(() => vi.fn())+const increment = vi.hoisted(() => vi.fn())+vi.mock('../src/services/adminSessionAuth.js', () => ({ authenticateAdminSession: authenticate }))+vi.mock('../src/services/rateLimiterService.js', async importOriginal => ({+ ...(await importOriginal<typeof import('../src/services/rateLimiterService.js')>()),+ checkAndIncrement: increment,+}))++const original = { codex: config.codexSubscriptionEnabled, grok: config.grokSubscriptionEnabled }+let session: string++function app() {+ const instance = express()+ instance.use('/api/v1', requireAuthForControlUI, createAdminSubscriptionCapabilitiesRouter())+ return instance+}++beforeEach(() => {+ // Only live database session state is substituted; quota attribution still+ // verifies the actual signed browser credential with the production verifier.+ session = signAdminToken('unit-admin')+ const claims = verifyAdminToken(session)+ authenticate.mockImplementation(async value => (value === session ? claims : null))+ increment.mockResolvedValue({+ allowed: true,+ remaining: 149,+ backendAvailable: true,+ count: 1,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ })+})++afterEach(() => {+ config.codexSubscriptionEnabled = original.codex+ config.grokSubscriptionEnabled = original.grok+ vi.clearAllMocks()+})++describe('administrative subscription capabilities', () => {+ it.each([+ [true, true],+ [true, false],+ [false, true],+ [false, false],+ ])('returns allowlisted integration flags for %s/%s', async (codex, grok) => {+ config.codexSubscriptionEnabled = codex+ config.grokSubscriptionEnabled = grok+ const response = await request(app())+ .get('/api/v1/admin/llm/providers/capabilities')+ .set('Cookie', `control_ui_admin_session=${session}`)+ .expect(200)+ expect(response.body).toEqual({+ providers: {+ 'codex-subscription': { enabled: codex },+ 'grok-subscription': { enabled: grok },+ },+ })+ expect(response.headers['cache-control']).toBe('private, no-store')+ expect(response.headers['x-ratelimit-limit']).toBe('150')+ })++ it('rejects missing or unverified sessions before capability discovery', async () => {+ await request(app()).get('/api/v1/admin/llm/providers/capabilities').expect(401)+ await request(app())+ .get('/api/v1/admin/llm/providers/capabilities')+ .set('Cookie', 'control_ui_admin_session=unverified-fixture')+ .expect(401)+ await request(app())+ .get('/api/v1/admin/llm/providers/capabilities')+ .set('Authorization', `Bearer ${session}`)+ .expect(401)+ expect(increment).not.toHaveBeenCalled()+ })++ it('retains the rate limit for authenticated capability reads', async () => {+ increment.mockResolvedValue({+ allowed: false,+ remaining: 0,+ backendAvailable: true,+ count: 151,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ })+ const response = await request(app())+ .get('/api/v1/admin/llm/providers/capabilities')+ .set('Cookie', `control_ui_admin_session=${session}`)+ .expect(429)+ expect(response.body.code).toBe('rate_limited')+ expect(response.body.message).toMatch(/Try again/)+ expect(response.headers['retry-after']).toBeDefined()+ })+})diff --git a/control-api/test/routes.internalPluginWorkloadSdk.test.ts b/control-api/test/routes.internalPluginWorkloadSdk.test.tsindex 1ff7df43a..238514ac4 100644--- a/control-api/test/routes.internalPluginWorkloadSdk.test.ts+++ b/control-api/test/routes.internalPluginWorkloadSdk.test.ts@@ -1,4 +1,4 @@-import { beforeEach, describe, expect, it, vi } from 'vitest'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import jwt from 'jsonwebtoken'import request from 'supertest'@@ -20,11 +20,16 @@ vi.mock('../src/services/pluginWorkloadSdkDb.js', () => ({finalizePluginWorkloadSdkRevocation: (...args: unknown[]) => db.finalize(...args),}))-function sign(iss: 'wrc' | 'hcc'): string {+function sign(+ iss: 'wrc' | 'hcc',+ jti = `${iss}-revocation-test`,+ audience = 'control-api',+ subject = `${iss}-provisioner`+): string {return jwt.sign(- { iss, aud: 'control-api', sub: `${iss}-provisioner` },+ { iss, aud: audience, sub: subject },iss === 'wrc' ? config.internalControlJwtWrcHmacSecret : config.internalControlJwtHccHmacSecret,- { algorithm: 'HS256', expiresIn: 60, jwtid: `${iss}-revocation-test` }+ { algorithm: 'HS256', expiresIn: 60, jwtid: jti })}@@ -36,11 +41,21 @@ function app() {}describe('internal Plugin Workload SDK revocation', () => {+ const originalLimit = config.pluginSdkInternalRlPerMinbeforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))+ config.pluginSdkInternalRlPerMin = 600db.revoke.mockReset()db.finalize.mockReset()limiterQuery.mockReset()- limiterQuery.mockResolvedValue({ rows: [{ count: 1 }], rowCount: 1 })+ const counters = new Map<string, number>()+ limiterQuery.mockImplementation(async (_sql: string, params: unknown[]) => {+ const key = `${String(params[0])}|${String(params[1])}`+ const count = (counters.get(key) ?? 0) + 1+ counters.set(key, count)+ return { rows: [{ count }], rowCount: 1 }+ })db.revoke.mockResolvedValue({state: 'revoking',revocationId: '11111111-1111-4111-8111-111111111111',@@ -56,6 +71,11 @@ describe('internal Plugin Workload SDK revocation', () => {})})+ afterEach(() => {+ config.pluginSdkInternalRlPerMin = originalLimit+ vi.useRealTimers()+ })+it('revoke and finalize carry the authenticated WRC technical principal', async () => {const authorization = `Bearer ${sign('wrc')}`const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }@@ -106,4 +126,114 @@ describe('internal Plugin Workload SDK revocation', () => {expect(response.status).toBe(403)expect(db.revoke).not.toHaveBeenCalled()})++ it('keeps six hundred forbidden HCC requests on their signed principal without stealing the WRC allowance on the same IP', async () => {+ const instance = app()+ const hcc = `Bearer ${sign('hcc')}`+ const wrc = `Bearer ${sign('wrc')}`+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }+ for (let index = 0; index < 600; index += 1) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', hcc)+ .send(binding)+ .expect(403)+ }+ expect(db.revoke).not.toHaveBeenCalled()+ for (let index = 0; index < 600; index += 1) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', wrc)+ .send(binding)+ .expect(200)+ }+ expect(db.revoke).toHaveBeenCalledTimes(600)+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', wrc)+ .send(binding)+ .expect(429)+ }, 15_000)++ it('keeps an exhausted invalid-audience or anonymous IP budget separate from the valid WRC allowance', async () => {+ const instance = app()+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }+ const invalidAudience = `Bearer ${sign('wrc', 'audience-negative', 'other-service')}`+ for (let index = 0; index < 600; index += 1) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', invalidAudience)+ .send(binding)+ .expect(401)+ }+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .send(binding)+ .expect(429)+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', `Bearer ${sign('wrc')}`)+ .send(binding)+ .expect(200)+ expect(db.revoke).toHaveBeenCalledTimes(1)+ }, 15_000)++ it('counts rotated signed credentials under one principal and recovers with PG at the minute boundary', async () => {+ config.pluginSdkInternalRlPerMin = 2+ const instance = app()+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }+ for (let index = 0; index < 2; index += 1) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', `Bearer ${sign('wrc', `rotation-${index}`)}`)+ .send(binding)+ .expect(200)+ }+ const denied = await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', `Bearer ${sign('wrc', 'rotation-third')}`)+ .send(binding)+ expect(denied.status).toBe(429)+ expect(denied.headers['retry-after']).toBe('43')+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', `Bearer ${sign('wrc', 'rotation-next-minute')}`)+ .send(binding)+ .expect(200)+ expect(db.revoke).toHaveBeenCalledTimes(3)+ })++ it('does not cap a higher verified internal override at the legacy IP ceiling of six hundred', async () => {+ config.pluginSdkInternalRlPerMin = 601+ const instance = app()+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }+ const authorization = `Bearer ${sign('wrc')}`+ for (let index = 0; index < 601; index += 1) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', authorization)+ .send(binding)+ .expect(200)+ }+ expect(db.revoke).toHaveBeenCalledTimes(601)+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', authorization)+ .send(binding)+ .expect(429)+ }, 15_000)++ it('keeps signed non-WRC subjects forbidden without turning an unusual subject into a server error', async () => {+ const instance = app()+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }+ for (const subject of ['service:ip:other', '\ud800']) {+ await request(instance)+ .post('/api/v1/internal/plugin-workload-sdk/revoke')+ .set('Authorization', `Bearer ${sign('hcc', 'subject-negative', 'control-api', subject)}`)+ .send(binding)+ .expect(403)+ }+ expect(db.revoke).not.toHaveBeenCalled()+ })})diff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.tsindex f50d28cf1..ac19373b0 100644--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts@@ -1,6 +1,7 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import request from 'supertest'+import { config } from '../src/config.js'import { rateLimitPool } from '../src/db.js'import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'import * as codexConnection from '../src/services/codexSubscriptionConnection.js'@@ -1453,7 +1454,10 @@ describe('POST /mcp-host/plugin-workload-sdk/credential-ticket/introspect', () =// second query so this test remains focused on ticket work protection.vi.mocked(rateLimitPool.query).mockResolvedValueOnce({ rows: [{ count: 1 }], rowCount: 1 } as never)- .mockResolvedValueOnce({ rows: [{ count: 121 }], rowCount: 1 } as never)+ .mockResolvedValueOnce({+ rows: [{ count: config.pluginSdkCredentialRlPerMin + 1 }],+ rowCount: 1,+ } as never)const res = await request(buildApp()).post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect').set('Authorization', `Bearer ${issueSdkToken()}`)diff --git a/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.ts b/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.tsnew file mode 100644index 000000000..ec108f298--- /dev/null+++ b/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.ts@@ -0,0 +1,220 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'+import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'+import { issuePluginWorkloadSdkCredentialTicket } from '../src/services/pluginWorkloadSdkCredentialTicket.js'+import { hashPromptTargetPolicy } from '../src/services/pluginWorkloadSdkDb.js'+import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'++const boundary = vi.hoisted(() => ({+ counts: new Map<string, number>(),+ reissue: vi.fn(),+ invocation: vi.fn(),+ grant: vi.fn(),+ receipt: vi.fn(),+ attempt: vi.fn(),+ redeem: vi.fn(),+}))++// The controlled limit exercises the real route guards at their configuration+// boundary. Only database state and ticket-authorizer persistence are mocked;+// JWT authentication, signed tickets, binding checks and quota middleware run.+vi.mock('../src/config.js', async importOriginal => {+ const actual = await importOriginal<typeof import('../src/config.js')>()+ return { ...actual, config: { ...actual.config, pluginSdkCredentialRlPerMin: 6 } }+})+vi.mock('../src/db.js', () => ({+ pool: { query: vi.fn(), connect: vi.fn() },+ withTransaction: vi.fn(),+ rateLimitPool: {+ query: vi.fn(async (_sql: string, params: unknown[]) => {+ const key = `${String(params[0])}|${String(params[1])}`+ const count = (boundary.counts.get(key) ?? 0) + 1+ boundary.counts.set(key, count)+ return { rows: [{ count }], rowCount: 1 }+ }),+ },+}))+vi.mock('../src/services/pluginWorkloadSdkDb.js', async importOriginal => {+ const actual = await importOriginal<typeof import('../src/services/pluginWorkloadSdkDb.js')>()+ return {+ ...actual,+ getInvocationById: (...args: unknown[]) => boundary.invocation(...args),+ findGrant: (...args: unknown[]) => boundary.grant(...args),+ getPluginWorkloadSdkAttemptReceipt: (...args: unknown[]) => boundary.receipt(...args),+ getPluginWorkloadSdkProviderAttempt: (...args: unknown[]) => boundary.attempt(...args),+ redeemPluginWorkloadSdkCredentialTicketJti: (...args: unknown[]) => boundary.redeem(...args),+ }+})+vi.mock('../src/services/pluginWorkloadSdkAuthorizer.js', async importOriginal => {+ const actual =+ await importOriginal<typeof import('../src/services/pluginWorkloadSdkAuthorizer.js')>()+ return {+ ...actual,+ reissuePromptBridgeCredentialTicket: (...args: unknown[]) => boundary.reissue(...args),+ }+})++const NS = 'sandbox-recipes'+const RECIPE = 'credential-budget'+const target = {+ targetRef: 'primary-zai',+ provider: 'zai',+ model: 'glm-4.7',+ credentialSlot: 'zai-api-key',+}+const policy = { policyRevision: 1, defaultTargetRef: target.targetRef, promptTargets: [target] }+const policyHash = hashPromptTargetPolicy(policy)+const attemptIds = [+ '11111111-1111-4111-8111-111111111111',+ '22222222-2222-4222-8222-222222222222',+ '33333333-3333-4333-8333-333333333333',+]++describe('SDK credential-operation budget across actual endpoints', () => {+ beforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))+ boundary.counts.clear()+ const used = new Set<string>()+ boundary.redeem.mockReset().mockImplementation(async ({ jti }: { jti: string }) => {+ if (used.has(jti)) return false+ used.add(jti)+ return true+ })+ boundary.grant.mockReset().mockResolvedValue({+ id: 'grant-credential-budget',+ recipeNamespace: NS,+ recipeName: RECIPE,+ capabilityFamily: 'promptBridge',+ ...policy,+ policyState: 'active',+ })+ boundary.invocation.mockReset().mockImplementation(async (id: string) => ({+ id,+ recipeNamespace: NS,+ recipeName: RECIPE,+ method: 'promptBridge',+ status: 'in_progress',+ attemptGeneration: 1,+ authorizationDecision: 'authorized',+ promptAuthorization: {+ policyRevision: 1,+ policyHash,+ authorizedTargetRefs: [target.targetRef],+ },+ }))+ boundary.receipt.mockReset().mockImplementation(async (invocationId: string) => ({+ invocationId,+ recipeNamespace: NS,+ recipeName: RECIPE,+ method: 'promptBridge',+ status: 'in_progress',+ attemptGeneration: 1,+ }))+ boundary.attempt.mockReset().mockImplementation(async (id: string) => ({+ id,+ invocationId: `inv-${attemptIds.indexOf(id) + 1}`,+ attemptGeneration: 1,+ targetRef: target.targetRef,+ status: 'in_progress',+ }))+ boundary.reissue+ .mockReset()+ .mockImplementation(async ({ invocationId }: { invocationId: string }) => {+ const index = Number(invocationId.slice(4)) - 1+ return {+ ok: true,+ value: {+ invocationId,+ attemptGeneration: 1,+ targetRef: target.targetRef,+ providerAttemptId: attemptIds[index],+ providerAttemptIndex: 1,+ credentialTicket: issuePluginWorkloadSdkCredentialTicket({+ recipeNamespace: NS,+ recipeName: RECIPE,+ invocationId,+ attemptGeneration: 1,+ providerAttemptId: attemptIds[index],+ providerAttemptIndex: 1,+ target,+ policyRevision: 1,+ policyHash,+ }),+ policyRevision: 1,+ policyHash,+ expiresInSeconds: 60,+ },+ }+ })+ })++ afterEach(() => vi.useRealTimers())++ it('permits two three-operation static credential sequences at six, then rejects the third reissue before ticket work', async () => {+ const instance = express()+ instance.use(express.json())+ instance.use(createMcpHostPluginWorkloadSdkRoutes())+ const authorization = `Bearer ${+ issueMcpHostAccessJwt(NS, RECIPE, undefined, {+ workflowControlScopes: ['plugin-workload-sdk'],+ }).token+ }`+ for (let index = 1; index <= 2; index += 1) {+ const invocationId = `inv-${index}`+ const issued = await request(instance)+ .post('/mcp-host/plugin-workload-sdk/prompt-bridge/credential-ticket')+ .set('Authorization', authorization)+ .send({+ recipeNamespace: NS,+ recipeName: RECIPE,+ invocationId,+ targetRef: target.targetRef,+ attemptGeneration: 1,+ })+ expect(issued.status).toBe(201)+ const body = {+ credentialTicket: issued.body.credentialTicket,+ invocationId,+ targetRef: target.targetRef,+ attemptGeneration: 1,+ providerAttemptId: attemptIds[index - 1],+ providerAttemptIndex: 1,+ }+ await request(instance)+ .post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect')+ .set('Authorization', authorization)+ .send({ ...body, redeem: false })+ .expect(200)+ expect(boundary.redeem).toHaveBeenCalledTimes(index - 1)+ await request(instance)+ .post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect')+ .set('Authorization', authorization)+ .send({ ...body, redeem: true })+ .expect(200)+ expect(boundary.redeem).toHaveBeenCalledTimes(index)+ }+ const denied = await request(instance)+ .post('/mcp-host/plugin-workload-sdk/prompt-bridge/credential-ticket')+ .set('Authorization', authorization)+ .send({+ recipeNamespace: NS,+ recipeName: RECIPE,+ invocationId: 'inv-3',+ targetRef: target.targetRef,+ attemptGeneration: 1,+ })+ expect(denied.status).toBe(429)+ expect(Number(denied.headers['retry-after'])).toBeGreaterThan(0)+ expect(boundary.reissue).toHaveBeenCalledTimes(2)+ expect(boundary.invocation).toHaveBeenCalledTimes(4)+ expect(boundary.redeem).toHaveBeenCalledTimes(2)+ const credentialCounts = [...boundary.counts.entries()].filter(([key]) =>+ key.startsWith('plugin_workload_sdk_credential:')+ )+ expect(credentialCounts).toHaveLength(1)+ expect(credentialCounts[0][0]).toContain(`plugin_workload_sdk_credential:${NS}/${RECIPE}|`)+ expect(credentialCounts[0][1]).toBe(7)+ })+})diff --git a/control-api/test/routes.registryConnect.test.ts b/control-api/test/routes.registryConnect.test.tsindex 07b681dad..e540db6d5 100644--- a/control-api/test/routes.registryConnect.test.ts+++ b/control-api/test/routes.registryConnect.test.ts@@ -8,6 +8,9 @@ import { checkAndIncrement } from '../src/services/rateLimiterService.js'const { cfg } = vi.hoisted(() => ({cfg: {+ adminRegistryConnectStatusPerMin: 150,+ adminRegistryConnectRequestPerMin: 15,+ adminRegistryConnectRecoveryPerMin: 50,registryConnectionMode: 'self-hosted',registryUrl: 'https://example.com',registryAuthEnabled: true,@@ -513,8 +516,8 @@ describe('registry connect flow', () => {connDb.getRegistryConnection.mockResolvedValue(null)const res = await request(app()).get('/admin/registry/connect').expect(200)expect(res.body.state).toBe('disconnected')- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_status:admin-uuid-123', 30)- expect(res.headers['x-ratelimit-limit']).toBe('30')+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_status:admin-uuid-123', 150)+ expect(res.headers['x-ratelimit-limit']).toBe('150')})// Proves the limiter blocks BEFORE any DB/registry work: a denied admin@@ -549,8 +552,8 @@ describe('registry connect flow', () => {.send({ requested_org_name: 'acme', contact_email: '[email protected]' }).expect(202)expect(res.body.state).toBe('pending')- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_request:admin-uuid-123', 3)- expect(res.headers['x-ratelimit-limit']).toBe('3')+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_request:admin-uuid-123', 15)+ expect(res.headers['x-ratelimit-limit']).toBe('15')})// Proves the limiter blocks BEFORE any registry work: a denied admin must@@ -1018,8 +1021,8 @@ describe('POST recover', () => {})const res = await request(app()).post('/admin/registry/connect/recover').expect(200)expect(res.body).toMatchObject({ state: 'connected', org: 'acme' })- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_recover:admin-uuid-123', 10)- expect(res.headers['x-ratelimit-limit']).toBe('10')+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_recover:admin-uuid-123', 50)+ expect(res.headers['x-ratelimit-limit']).toBe('50')})// Proves the limiter actually blocks, and blocks BEFORE any registry work:diff --git a/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.ts b/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.tsnew file mode 100644index 000000000..e96dd8a6e--- /dev/null+++ b/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.ts@@ -0,0 +1,232 @@+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'+import { createHash, randomBytes } from 'node:crypto'+import { Pool, type PoolClient } from 'pg'+import { applyAdminSubscriptionRateLimitNamespace } from '../src/services/adminSubscriptionRateLimitMigration.js'+import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++describeRealPostgres('administrative subscription counter namespace migration', () => {+ const database = `admin_quota_migration_${randomBytes(6).toString('hex')}`+ let adminPool: Pool+ let pool: Pool+ let client: PoolClient+ const windowStart = 1_800_000_000_000+ const prefixes = [+ ['admin_codex_read:', 'admin_subscription_read:'],+ ['admin_codex_write:', 'admin_subscription_write:'],+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],+ ] as const++ beforeAll(async () => {+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE "${database}"`)+ const url = new URL(adminUrl!)+ url.pathname = `/${database}`+ pool = new Pool({ connectionString: url.toString() })+ client = await pool.connect()+ }, 60_000)++ beforeEach(async () => {+ await client.query('DROP TABLE IF EXISTS rate_limit_buckets CASCADE')+ await client.query(`CREATE TABLE rate_limit_buckets (+ bucket_key TEXT NOT NULL,+ window_start_ms BIGINT NOT NULL,+ count INTEGER NOT NULL DEFAULT 0,+ PRIMARY KEY (bucket_key, window_start_ms)+ )`)+ })++ afterAll(async () => {+ client?.release()+ await pool?.end()+ if (!adminPool) return+ await adminPool.query(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`)+ await adminPool.end()+ })++ async function migrate() {+ await client.query('BEGIN')+ try {+ await applyAdminSubscriptionRateLimitNamespace(client)+ await client.query('COMMIT')+ } catch (error) {+ await client.query('ROLLBACK')+ throw error+ }+ }++ it.each(prefixes)(+ 'merges existing %s accounting without a new allowance',+ async (legacy, current) => {+ // Synthetic suffixes model ledger identities, not authentication tokens.+ await client.query(+ `INSERT INTO rate_limit_buckets VALUES+ ($1, $3, 31), ($2, $3, 9), ('unrelated:fixture', $3, 7)`,+ [`${legacy}state:fixture`, `${current}state:fixture`, windowStart]+ )+ await migrate()+ const rows = await client.query(+ 'SELECT bucket_key, count FROM rate_limit_buckets ORDER BY bucket_key'+ )+ expect(rows.rows).toEqual([+ { bucket_key: `${current}state:fixture`, count: 40 },+ { bucket_key: 'unrelated:fixture', count: 7 },+ ])+ await migrate()+ const repeated = await client.query(+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1',+ [`${current}state:fixture`]+ )+ expect(repeated.rows[0].count).toBe(40)+ }+ )++ it.each(prefixes)('normalizes writes from an older %s binary', async (legacy, current) => {+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 31)', [+ `${legacy}fixture`,+ windowStart,+ ])+ await migrate()+ const check = await checkAndIncrementWithQuery(+ (sql, values) => client.query(sql, values),+ `${legacy}fixture`,+ 150,+ windowStart+ )+ expect(check.count).toBe(32)+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')+ expect(rows.rows).toEqual([{ bucket_key: `${current}fixture`, count: 32 }])+ })++ it('keeps concurrent old and new writers on one physical counter', async () => {+ await migrate()+ const results = await Promise.all(+ Array.from({ length: 40 }, (_, index) =>+ checkAndIncrementWithQuery(+ (sql, values) => pool.query(sql, values),+ `${index % 2 ? 'admin_codex_read:' : 'admin_subscription_read:'}fixture`,+ 32,+ windowStart+ )+ )+ )+ expect(results.filter(result => result.allowed)).toHaveLength(32)+ expect(new Set(results.map(result => result.count)).size).toBe(40)+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')+ expect(rows.rows).toEqual([{ bucket_key: 'admin_subscription_read:fixture', count: 40 }])+ })++ it.each(prefixes)(+ 'retains opaque long-key accounting for %s',+ async (legacyPrefix, currentPrefix) => {+ const suffix = `state:${'x'.repeat(600)}`+ const legacy = legacyPrefix + suffix+ const current = currentPrefix + suffix+ const digest = `sha256-long-key:${createHash('sha256').update(legacy).digest('hex')}`+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $3, 31), ($2, $3, 9)', [+ digest,+ current,+ windowStart,+ ])+ await migrate()+ const renamed = await checkAndIncrementWithQuery(+ (sql, values) => client.query(sql, values),+ current,+ 150,+ windowStart+ )+ expect(renamed.count).toBe(41)+ const older = await checkAndIncrementWithQuery(+ (sql, values) => client.query(sql, values),+ digest,+ 150,+ windowStart+ )+ expect(older.count).toBe(42)+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')+ expect(rows.rows).toEqual([{ bucket_key: digest, count: 42 }])+ }+ )++ it('recovers at the next actual SQL window without deleting consumed counts', async () => {+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 150)', [+ 'admin_codex_read:fixture',+ windowStart,+ ])+ await migrate()+ const run = (now: number) =>+ checkAndIncrementWithQuery(+ (sql, values) => client.query(sql, values),+ 'admin_subscription_read:fixture',+ 150,+ now+ )+ expect((await run(windowStart)).allowed).toBe(false)+ const next = await run(windowStart + 60_000)+ expect(next.allowed).toBe(true)+ expect(next.count).toBe(1)+ const previous = await client.query(+ 'SELECT count FROM rate_limit_buckets WHERE window_start_ms = $1',+ [windowStart]+ )+ expect(previous.rows[0].count).toBe(151)+ })++ it('retains exact suffixes and independently counted windows', async () => {+ const suffix = 'state:fixture_%:admin_codex_read:embedded'+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 4), ($1, $3, 6)', [+ `admin_codex_read:${suffix}`,+ windowStart,+ windowStart + 60_000,+ ])+ await migrate()+ const rows = await client.query(+ 'SELECT bucket_key, window_start_ms, count FROM rate_limit_buckets ORDER BY window_start_ms'+ )+ expect(rows.rows).toEqual([+ {+ bucket_key: `admin_subscription_read:${suffix}`,+ window_start_ms: String(windowStart),+ count: 4,+ },+ {+ bucket_key: `admin_subscription_read:${suffix}`,+ window_start_ms: String(windowStart + 60_000),+ count: 6,+ },+ ])+ })++ it('preserves accounting when the longer namespace crosses the 512-byte boundary', async () => {+ const suffix = 'x'.repeat(495)+ const legacy = 'admin_codex_read:' + suffix+ const current = 'admin_subscription_read:' + suffix+ expect(Buffer.byteLength(legacy)).toBe(512)+ expect(Buffer.byteLength(current)).toBeGreaterThan(512)+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $3, 31), ($2, $3, 9)', [+ legacy,+ current,+ windowStart,+ ])+ await migrate()+ const expected = `sha256-long-key:${createHash('sha256').update(legacy).digest('hex')}`+ const result = await checkAndIncrementWithQuery(+ (sql, values) => client.query(sql, values),+ current,+ 150,+ windowStart+ )+ expect(result.count).toBe(41)+ // An older binary's short raw input is canonicalized by the invoker trigger.+ await client.query(+ `INSERT INTO rate_limit_buckets VALUES ($1, $2, 1)+ ON CONFLICT (bucket_key, window_start_ms) DO UPDATE+ SET count = rate_limit_buckets.count + EXCLUDED.count`,+ [legacy, windowStart]+ )+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')+ expect(rows.rows).toEqual([{ bucket_key: expected, count: 42 }])+ })+})diff --git a/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts b/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.tsindex f10ac6885..fd3feb5d4 100644--- a/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts+++ b/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts@@ -52,7 +52,7 @@ const grant = (// ─── Issue #348 (plan D3) — platform per-minute defaults ─────────────────//// RED-FIRST (plan §6.6): these tests assert the POST-change ceilings-// (promptBridge 120/min, clientNotifications 150/min, sourced from+// (promptBridge 600/min, clientNotifications 750/min, sourced from// config.pluginSdkPromptBridgeRlPerMin / config.pluginSdkNotificationsRlPerMin).// Each case is POSITIVE (at the ceiling → allowed) AND NEGATIVE (one over → denied// with quota_exceeded). Against pre-change code the hardcoded defaults are 60/120,@@ -71,32 +71,32 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =vi.mocked(sdkDb.countRecentInvocations).mockReset()})- it('allows promptBridge at exactly 120/minute (positive) and denies at 121 (negative, platform default)', async () => {+ it('allows promptBridge at exactly 600/minute (positive) and denies at 601 (negative, platform default)', async () => {const noOverrides = grant({ quotaLimits: {} })// POSITIVE: at the ceiling → allowed.- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(120)+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(600)await expect(checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)).resolves.toEqual({ ok: true })// NEGATIVE: one over the ceiling → denied.- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(121)+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(601)await expect(checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)).resolves.toMatchObject({ok: false,error: 'quota_exceeded',retryable: false,- message: expect.stringContaining('120/minute'),+ message: expect.stringContaining('600/minute'),})})- it('allows clientNotifications at exactly 150/minute (positive) and denies at 151 (negative), narrowed to the eventType', async () => {+ it('allows clientNotifications at exactly 750/minute (positive) and denies at 751 (negative), narrowed to the eventType', async () => {const noOverrides = grant({ capabilityFamily: 'clientNotifications', quotaLimits: {} })// POSITIVE: at the ceiling → allowed.- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(150)+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(750)await expect(checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', noOverrides, {eventType: 'lead.followup.due',@@ -110,7 +110,7 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =)// NEGATIVE: one over the ceiling → denied.- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(151)+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(751)await expect(checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', noOverrides, {eventType: 'lead.followup.due',@@ -119,14 +119,14 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =ok: false,error: 'quota_exceeded',retryable: false,- message: expect.stringContaining('150/minute'),+ message: expect.stringContaining('750/minute'),})})- it('frees the recipe once the trailing window drains (deny at 121, allow at 0)', async () => {+ it('frees the recipe once the trailing window drains (deny at 601, allow at 0)', async () => {const noOverrides = grant({ quotaLimits: {} })- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(121)+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(601)await expect(checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)).resolves.toMatchObject({ ok: false, error: 'quota_exceeded' })@@ -158,4 +158,29 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =message: expect.stringContaining('5/minute'),})})++ it('lets a grant-level maxNotificationsPerMinute override win over the platform default', async () => {+ const withOverride = grant({+ capabilityFamily: 'clientNotifications',+ quotaLimits: { maxNotificationsPerMinute: 9 },+ })++ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(9)+ await expect(+ checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', withOverride, {+ eventType: 'lead.followup.due',+ })+ ).resolves.toEqual({ ok: true })++ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(10)+ await expect(+ checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', withOverride, {+ eventType: 'lead.followup.due',+ })+ ).resolves.toMatchObject({+ ok: false,+ error: 'quota_exceeded',+ message: expect.stringContaining('9/minute'),+ })+ })})diff --git a/control-api/test/services.rateLimitBucketKey.test.ts b/control-api/test/services.rateLimitBucketKey.test.tsnew file mode 100644index 000000000..5f99eaab3--- /dev/null+++ b/control-api/test/services.rateLimitBucketKey.test.ts@@ -0,0 +1,47 @@+import { describe, expect, it } from 'vitest'+import { createHash } from 'node:crypto'+import {+ MAX_RATE_LIMIT_BUCKET_KEY_BYTES,+ boundedBucketKey,+} from '../src/services/rateLimitBucketKey.js'++describe('rate-limit bucket storage identity', () => {+ const aliases = [+ ['admin_codex_read:', 'admin_subscription_read:'],+ ['admin_codex_write:', 'admin_subscription_write:'],+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],+ ] as const++ it.each(aliases)(+ 'preserves the opaque legacy %s identity for long renamed keys',+ (oldPrefix, newPrefix) => {+ const suffix = 'fixture_'.repeat(100)+ const expected = `sha256-long-key:${createHash('sha256')+ .update(oldPrefix + suffix)+ .digest('hex')}`+ expect(boundedBucketKey(oldPrefix + suffix)).toBe(expected)+ expect(boundedBucketKey(newPrefix + suffix)).toBe(expected)+ }+ )++ it('keeps unrelated and embedded namespace text unchanged', () => {+ const key = 'unrelated:admin_subscription_read:' + 'fixture_'.repeat(100)+ expect(boundedBucketKey(key)).toBe(+ `sha256-long-key:${createHash('sha256').update(key).digest('hex')}`+ )+ })++ it('keeps already bounded neutral namespaces for the database migration', () => {+ const key = 'admin_subscription_read:fixture'+ expect(boundedBucketKey(key)).toBe(key)+ })++ it('uses the byte boundary and keeps stored keys bounded', () => {+ const oldKey = 'admin_codex_read:' + 'é'.repeat(MAX_RATE_LIMIT_BUCKET_KEY_BYTES)+ const newKey = 'admin_subscription_read:' + 'é'.repeat(MAX_RATE_LIMIT_BUCKET_KEY_BYTES)+ expect(boundedBucketKey(newKey)).toBe(boundedBucketKey(oldKey))+ expect(Buffer.byteLength(boundedBucketKey(newKey))).toBeLessThan(+ MAX_RATE_LIMIT_BUCKET_KEY_BYTES+ )+ })+})diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.tsindex 1d6960666..a6f2028aa 100644--- a/control-api/test/workflows.shared.rateLimit.test.ts+++ b/control-api/test/workflows.shared.rateLimit.test.ts@@ -1,19 +1,20 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { createHash } from 'node:crypto'import request from 'supertest'import { config } from '../src/config.js'import {- adminCodexReadRateLimits,- adminCodexWriteRateLimits,adminOutputsReadRateLimits,+ adminSubscriptionReadRateLimits,+ adminSubscriptionWriteRateLimits,adminWorkflowRateLimitCredential,adminWorkflowTriggerRateLimit,- codexOAuthCallbackRateLimits,llmProviderAttemptAuthorizeRateLimits,mcpHostAttemptRateLimitKey,mcpHostWorkflowTriggerRateLimit,mcpHostWorkflowTriggerRateLimitCredential,shouldSkipWorkflowGrantEdgeRateLimit,+ subscriptionOAuthCallbackRateLimits,verifiedAdminRateLimitSubject,workflowAdminReadRateLimits,workflowGrantEdgeRateLimitKey,@@ -99,7 +100,7 @@ describe('routes/workflows/shared/rateLimit', () => {const counts = countRequests()const app = express()for (const path of ['/codex/connections', '/grok/connections', '/grok/models']) {- app.get(path, ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ app.get(path, ...adminSubscriptionReadRateLimits(), (_req, res) => res.sendStatus(204))}const paths = ['/codex/connections', '/grok/connections', '/grok/models']for (let i = 0; i < 150; i++) {@@ -126,7 +127,9 @@ describe('routes/workflows/shared/rateLimit', () => {it('gives distinct verified sessions independent subscription allowances', async () => {countRequests()const app = express()- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )for (const session of ['signed-admin-a', 'signed-admin-b']) {for (let i = 0; i < 150; i++) {await request(app)@@ -141,13 +144,37 @@ describe('routes/workflows/shared/rateLimit', () => {.expect(429)})+ it('aligns edge recovery with the database wall-clock window after a late first request', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(1_800_000_030_000)+ mockVerifyAdminToken.mockImplementation(() => signedClaims('window-fixture'))+ countRequests()+ const app = express()+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )+ const cookie = 'control_ui_admin_session=window-fixture'+ const first = await request(app).get('/connections').set('Cookie', cookie).expect(204)+ expect(first.headers.ratelimit).toMatch(/reset=30\b/)+ for (let i = 1; i < 150; i++)+ await request(app).get('/connections').set('Cookie', cookie).expect(204)+ const denied = await request(app).get('/connections').set('Cookie', cookie).expect(429)+ expect(denied.headers['retry-after']).toBe('30')+ vi.setSystemTime(1_800_000_060_000)+ for (let i = 0; i < 150; i++)+ await request(app).get('/connections').set('Cookie', cookie).expect(204)+ await request(app).get('/connections').set('Cookie', cookie).expect(429)+ })+it('uses a configured read budget for both edge and distributed gates', async () => {const original = config.adminSubscriptionReadPerMinconfig.adminSubscriptionReadPerMin = 3try {countRequests()const app = express()- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )for (let i = 0; i < 3; i++) {await request(app).get('/connections')@@ -165,11 +192,36 @@ describe('routes/workflows/shared/rateLimit', () => {}})+ it.each(['signed-admin-b', 'unverified-fixture'])(+ 'attributes subscription reads to the cookie despite an extra bearer %s',+ async extra => {+ countRequests()+ const app = express()+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )+ await request(app)+ .get('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .set('Authorization', `Bearer ${extra}`)+ .expect(204)+ const expected = createHash('sha256').update('signed-admin-a').digest('hex').slice(0, 32)+ expect(mockCheckAndIncrement.mock.calls[0]?.slice(0, 2)).toEqual([+ `admin_subscription_read:${expected}`,+ 150,+ ])+ }+ )+it('allows 100 subscription writes without consuming the read quota', async () => {const counts = countRequests()const app = express()- app.post('/connections', ...adminCodexWriteRateLimits(), (_req, res) => res.sendStatus(204))- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ app.post('/connections', ...adminSubscriptionWriteRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )for (let i = 0; i < 100; i++) {await request(app).post('/connections')@@ -190,7 +242,9 @@ describe('routes/workflows/shared/rateLimit', () => {it('enforces the 100 callback budget with retry timing', async () => {countRequests()const app = express()- app.get('/callback', ...codexOAuthCallbackRateLimits(), (_req, res) => res.sendStatus(204))+ app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )for (let i = 0; i < 100; i++) {await request(app).get('/callback').query({ state: 'unit-callback-state' }).expect(204)}@@ -202,6 +256,40 @@ describe('routes/workflows/shared/rateLimit', () => {expect(denied.headers['retry-after']).toBeDefined()})+ it.each([+ ['workflow grant reads', 300, workflowGrantReadRateLimits],+ ['workflow grant writes', 100, workflowGrantWriteRateLimits],+ ['workflow administrative reads', 300, workflowAdminReadRateLimits],+ ['administrative output reads', 150, adminOutputsReadRateLimits],+ ] as const)(+ 'allows the increased %s capacity through both gates',+ async (_family, limit, factory) => {+ mockVerifyAdminToken.mockImplementation(() => signedClaims('capacity-fixture'))+ countRequests()+ const app = express()+ app.get('/capacity', ...factory(), (_req, res) => res.sendStatus(204))+ const cookie = 'control_ui_admin_session=capacity-fixture'+ for (let i = 0; i < limit; i++) {+ await request(app).get('/capacity').set('Cookie', cookie).expect(204)+ }+ const denied = await request(app).get('/capacity').set('Cookie', cookie).expect(429)+ expect(denied.body.code).toBe('rate_limited')+ expect(denied.headers['retry-after']).toBeDefined()+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === limit)).toBe(true)+ }+ )++ it('retains the 20-request callback IP safeguard when no state is supplied', async () => {+ countRequests()+ const app = express()+ app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>+ res.sendStatus(204)+ )+ for (let i = 0; i < 20; i++) await request(app).get('/callback').expect(204)+ await request(app).get('/callback').expect(429)+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === 20)).toBe(true)+ })+beforeEach(() => {mockVerifyAdminToken.mockImplementation((token: string) =>token.startsWith('signed-') ? signedClaims(token.slice('signed-'.length)) : null@@ -363,9 +451,9 @@ describe('routes/workflows/shared/rateLimit', () => {expect(workflowGrantWriteRateLimits()).toHaveLength(2)expect(workflowAdminReadRateLimits()).toHaveLength(2)expect(adminOutputsReadRateLimits()).toHaveLength(2)- expect(adminCodexReadRateLimits()).toHaveLength(2)- expect(adminCodexWriteRateLimits()).toHaveLength(2)- expect(codexOAuthCallbackRateLimits()).toHaveLength(2)+ expect(adminSubscriptionReadRateLimits()).toHaveLength(2)+ expect(adminSubscriptionWriteRateLimits()).toHaveLength(2)+ expect(subscriptionOAuthCallbackRateLimits()).toHaveLength(2)expect(llmProviderAttemptAuthorizeRateLimits()).toHaveLength(2)})diff --git a/deploy/base/control-plane/configmaps.yaml b/deploy/base/control-plane/configmaps.yamlindex 4ef42b1ec..75434f139 100644--- a/deploy/base/control-plane/configmaps.yaml+++ b/deploy/base/control-plane/configmaps.yaml@@ -60,18 +60,44 @@ data:WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_ENABLED: "true"WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_INTERVAL_MS: "5000"WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_BATCH_SIZE: "10"- APPROVAL_RL_REQUEST_PER_MIN: "120"+ APPROVAL_RL_REQUEST_PER_MIN: "600"+ APPROVAL_RL_REFRESH_PER_MIN: "100"+ APPROVAL_RL_REISSUE_PER_MIN: "25"+ CONTROL_API_OAUTH_BROKER_RL_PER_MIN: "300"+ CONTROL_API_HOST_WAKE_RL_PER_MIN: "150"# Plugin Workload SDK platform rate limits (issue #348). Per-run SDK quotas# (maxRequestsPerRun / maxNotificationsPerRun) are deprecated and ignored;- # limits are enforced over a trailing 60s window (no fixed reset); once- # traffic slows below the limit the window drains within ~60s — stepless- # recipes are never permanently trapped. Values MUST equal the code defaults- # in control-api/src/config.ts (150/120/600/600) — the config deploy-mirror- # test fails on drift.- CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "150"- CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "120"- CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "600"+ # Method quotas use their existing trailing 60s window. Authenticated request+ # and edge budgets use the PG calendar minute; anonymous IP backstops retain+ # first-hit windows. Defaults below mirror control-api/src/config.ts.+ # Healthy static prompts cost three credential operations and six/seven SDK+ # API requests; the 1800/6000 defaults keep the prompt600 allowance reachable+ # without removing fresh capability proof, either Secret check or redemption.+ CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "750"+ CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "6000"CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: "6000"+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: "1800"+ CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN: "300"+ CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN: "300"+ CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN: "100"+ CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN: "50"+ CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN: "150"+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN: "300"+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN: "60"+ CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN: "150"+ CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN: "150"+ CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN: "150"+ CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN: "15"+ CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN: "50"+ CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN: "300"+ CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN: "150"+ CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN: "150"+ CONTROL_API_ADMIN_GFS_SHARES_PER_MIN: "150"+ CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN: "150"# Hard ceiling for caller-provided ttlSeconds. Defaults to 48h so a client# asking for ttlSeconds=604800 (code default) can't freeze a workflow step# for a full week. Overlays may tighten further for stricter environments.diff --git a/deploy/overlays/minikube/configmaps/control-api-config.yaml b/deploy/overlays/minikube/configmaps/control-api-config.yamlindex 91c9fa842..21c7ba16b 100644--- a/deploy/overlays/minikube/configmaps/control-api-config.yaml+++ b/deploy/overlays/minikube/configmaps/control-api-config.yaml@@ -89,10 +89,14 @@ data:# trailing 60s window (no fixed reset); once traffic slows below the limit# the window drains within ~60s — stepless recipes are never permanently# trapped.- CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "150"- CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "120"- CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "750"+ CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "6000"CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: "6000"+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: "600"+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: "1800"# OSS default: point at the shared registry so a self-hoster browses the# public catalog out of the box and can run the connect flow (register →# approve → claim) against it. self-hosted is the only mode that fits an OSSdiff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.mdnew file mode 100644index 000000000..70c0ff2fa--- /dev/null+++ b/docs/architecture/administrative-rate-limits.md@@ -0,0 +1,147 @@+# Administrative rate limits and coordinated access++Control API owns request quotas. Control UI reduces redundant optional reads+and explains genuine throttling. A person’s action can generate several HTTP+requests, and each request can traverse several independent counters.++For one workload sharing the relevant principal and window, the action bound+is the minimum of each applicable budget divided by the operations that+action charges to it. A requests-per-minute setting is not a throughput+guarantee. Retries, failover and other methods consume additional allowance.++## Administrative browser path++Browser -> public ingress -> Control UI Next proxy -> Control API admin+authentication -> local edge guard -> shared PostgreSQL counter -> response.++The checked-in UI proxy has no requests-per-minute quota and forwards status,+body, Retry-After and quota headers. Checked-in Control UI calls Control API+directly; the repository’s nginx gateway configs contain no limit_req or+limit_conn on this path. Live Cloudflare/WAF policy, forwarding-IP trust and+actual deployed ENV owners remain deployment checks.++Subscription administration and capability discovery use the cookie accepted+by the parent administrator middleware. Quota attribution follows that same+cookie even when an unrelated Authorization header is present. Workflow+administration preserves its separate existing Bearer/cookie transport.++Read quotas are shared across subscription providers and capability discovery.+Writes have a separate quota. Neither family creates a global aggregate of+every administrator. Existing signed-session hashing and verified Host/recipe+bindings are preserved; a naming change does not choose a new identity policy.++## Configuration defaults++All values below count operations per minute unless stated otherwise. Valid+explicit numeric ENV and grant overrides remain authoritative; deliberately+lower overrides lower the effective workload capacity.++| Family | Configuration/default | Binding and inner coordination |+| --- | --- | --- |+| Subscription reads | CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN=150 | Same signed browser credential across providers/capabilities; aligned edge and ledger. |+| Subscription writes | CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN=100 | Separate from reads; no automatic replay. |+| Subscription callback | CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN=100 | Per supplied state; no-state IP safeguard20. Presence is not state validation. |+| Workflow administrative reads | CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN=300 | Existing verified administrator credential. |+| Workflow grant reads/writes | CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN=300; CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN=100 | Separate families and aligned verified edge/ledger. |+| Administrative workflow trigger | CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN=50 | Existing verified caller. External/MCP trigger contract remains10. |+| Outputs reads | CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN=150 | Existing administrator binding. |+| Host wake | CONTROL_API_HOST_WAKE_RL_PER_MIN=150 | Verified Host reference after authentication and Host match. |+| Registry keys/grants/status/connect/recovery | CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN=150; GRANTS=150; CONNECT_STATUS=150; CONNECT_REQUEST=15; CONNECT_RECOVERY=50 | Separate authenticated administrator-subject families; each suffix uses the CONTROL_API_ADMIN_REGISTRY_ prefix. |+| Connector deletion | CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN=300; CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN=150 | PostgreSQL150 controls the authenticated allowance. Authorization is preserved. |+| SDK admin/internal | CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN=600; CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN=600 | Verified principal; internal signed iss/sub separated from invalid/anonymous IP600. |+| SDK authenticated request/preauth | CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN=6000; CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN=6000 | Verified runtime Host/recipe; aligned calendar minute. |+| SDK anonymous preauth | CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN=600 | Source-IP first-hit backstop; a valid attributed runtime JWT does not consume it. |+| SDK credential operations | CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN=1800 | Reissue and both introspections share this quota. Ticket authority/binding and single use remain. |+| SDK prompt/notification defaults | CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN=600; CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN=750 | Explicit grant overrides remain; existing method-quota window semantics remain. |+| Approval request/refresh/reissue | APPROVAL_RL_REQUEST_PER_MIN=600; APPROVAL_RL_REFRESH_PER_MIN=100; APPROVAL_RL_REISSUE_PER_MIN=25 | Existing verified binding and token rotation semantics. |+| Provider attempt authorization/OAuth broker | CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN=300; CONTROL_API_OAUTH_BROKER_RL_PER_MIN=300 | Existing verified domain budgets; unverified attempt-authorization IP60 remains. |+| Administrative GFS grants/shares/report | CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN=150; CONTROL_API_ADMIN_GFS_SHARES_PER_MIN=150; CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN=150 | Existing operator permissions; grants/read-write and shares keep their own families. |++Code defaults and base/minikube ConfigMap literals must agree. A code-only+increase can be defeated by an older deployment override. This document and+changed manifests are source artifacts, not confirmation of a rollout.++## SDK operation multiplicity++A healthy static-secret attempt performs reissue, pre-Secret introspection and+post-Secret introspection/redemption. At the old 120 operation allowance this+guard admitted at most40 healthy attempts per minute. All three operations+are retained. 1800 operations make600 healthy single-attempt prompts possible+through this guard.++Each sequential prompt also requests fresh capability proof, authorizes and+closes its lifecycle. SDK-only closure totals six Control API requests; a+runtime with separate provider/invocation status closure totals seven. 600+healthy prompts therefore consume 3600/4200 general requests before bootstrap,+other methods or retries. Both authenticated request gates default to 6000.+Capability authority is not cached to reduce those charges.++The three credential requests carry the Host runtime access JWT, including+the requests forwarded by WRC. They are attributed to that verified Host or+recipe, not to WRC’s internal service identity. Namespace equal to the Hosts+namespace uses primary hostRefs[0]; other callers use namespace/recipeName.+Both paths retain signature, audience, expiry and normalized binding checks.++## Windows, headers and backend outages++Authenticated edge counters and the PostgreSQL ledger use calendar minutes.+A late first request no longer leaves an exhausted edge counter active after+the database reset. Server-generated source-IP keys retain the previous+first-hit abuse window. Always-counted calendar stores do not enable+skipSuccessfulRequests or skipFailedRequests.++The last enforcing ledger or process-memory counter owns draft7, draft6 and+legacy X quota metadata. 429 includes rate_limited, a readable message,+retryAfterSeconds and Retry-After. 503 unavailable removes stale quota metadata+and retains its retry contract; an unavailable count is not zero usage.++The existing PostgreSQL outage policy is unchanged. Process-memory counters+are per process and use first-hit 60 s windows with bounded key cardinality.+Their counters do not combine with PostgreSQL, and they are not a distributed+quota. Outage recovery can therefore differ from normal ledger enforcement.++## Naming and active accounting++Shared infrastructure uses capability names. The previous admin_codex_read,+admin_codex_write and codex_oauth_callback categories become subscription+categories; legitimate vendor protocol identifiers remain specific.++The namespace migration merges counts atomically inside the existing migration+transaction and preserves windows/suffixes. An invoker trigger normalizes old+writers during rolling updates and supported rollback. Existing bounded long+key digests and the 512-byte boundary remain compatible. Do not clear counters+to pass a smoke test. Retire the compatibility trigger only through a later+reviewed migration after old writers and rollback support are retired.++## Control UI recovery++Only allowlisted safe subscription metadata uses browser-local reuse: TTL 30 s,+maximum 128 entries, origin/principal/scope/session-epoch binding and a shared+subscription-read cooldown. Ordinary Agent sections do not fetch optional+subscription inventory; editor opening loads it when needed.++At most one shared idempotent recovery GET proceeds after a denial deadline.+Forced refresh respects the same pause. Authentication changes, 401 and+cross-tab invalidation fence old awaits and clear identity-bound state.+Aborting one subscriber does not cancel another subscriber’s shared request.+SSR does not retain cached/deduplicated results across requests.++Do not cache authentication decisions, general mutable reads, file contents,+OAuth polling or credentials. Mutations and OAuth operations are not replayed+automatically. Optional provider failures preserve independent Agent data,+known summaries and drafts; unknown availability is not disabled availability.++## Evidence and rollout boundary++Unit/build, real PostgreSQL, local T0/T1/T2, visible browser and deployed+receipts are distinct. Require actual selected tests, complete reports and+producer exit 0; skipped physical tests do not establish correctness.++For an authorized rollout, verify effective numeric owners, source/image+revision, ledger migration ordering, old-writer drain, independent identities,+forwarding policy and visible recovery. Keep upstream provider quotas, bytes,+concurrency and unrelated authentication/minting safeguards separate.429 must+remain possible under deliberate overload and must be understandable.++References: issue #985; #433 owns broader edge-store/cardinality coordination;+#438 owns changes to principal policy. Neither is silently replaced here.diff --git a/docs/crds/workflowrecipe.md b/docs/crds/workflowrecipe.mdindex d5aefdfcf..51899d793 100644--- a/docs/crds/workflowrecipe.md+++ b/docs/crds/workflowrecipe.md@@ -1377,12 +1377,12 @@ Declaring this block forces an always-on `mcp-host`. Runtime enforcement is addi| `promptBridge.maxOutputTokens` | integer | No | Min 1. Enforced as `max_tokens` on API-key providers (OpenAI, Claude, Z.AI, Bedrock, Google). `codex-subscription` cannot bind it: the ChatGPT wire rejects `max_output_tokens`, so there the ceiling only shapes the authorize request identity and the enforced bound is the contract limit of 16384. || `promptBridge.maxRequestsPerRun` | integer | No | Min 1. **Deprecated:** accepted but ignored; platform per-minute rate limits apply. Removal planned. || `promptBridge.maxConcurrentInvocations` | integer | No | Min 1. Runtime default 5. |-| `promptBridge.maxInvocationsPerMinute` | integer | No | Min 1. Platform default 120 when unset (ENV `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN`, issue #348). |+| `promptBridge.maxInvocationsPerMinute` | integer | No | Min 1. Platform default 600 when unset (ENV `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN`, issue #348). || `clientNotifications.allowedEventTypes` | string[] | **Yes**\† | 1–64 entries. || `clientNotifications.allowedTargetRefs` | string[] | No | Max 64. || `clientNotifications.allowedUserRefs` | boolean | No | Whether `userRef` targets are permitted. || `clientNotifications.maxNotificationsPerRun` | integer | No | Min 1. **Deprecated:** accepted but ignored; platform per-minute rate limits apply. Removal planned. |-| `clientNotifications.maxNotificationsPerMinute` | integer | No | Min 1. Platform default 150 when unset (ENV `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN`, issue #348). |+| `clientNotifications.maxNotificationsPerMinute` | integer | No | Min 1. Platform default 750 when unset (ENV `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN`, issue #348). || `allowedCallers` | string[] | No | Workload ids permitted to call the SDK. **Empty = all declared workloads.** || `idempotencyKeyPattern` | string | No | Regex; runtime default `^[a-zA-Z0-9_-]{1,128}$`. Must compile. |diff --git a/docs/deploy/plugin-workload-sdk-upgrade.md b/docs/deploy/plugin-workload-sdk-upgrade.mdindex ab3b378fa..15fe1564c 100644--- a/docs/deploy/plugin-workload-sdk-upgrade.md+++ b/docs/deploy/plugin-workload-sdk-upgrade.md@@ -63,10 +63,11 @@ permanently trapped.| ENV variable | Default | Limits ||---|---|---|-| `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN` | `150` | clientNotifications invocations per minute |-| `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN` | `120` | promptBridge invocations per minute |-| `CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN` | `600` | shared SDK request bucket per minute |-| `CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN` | `600` | pre-authorization checks per minute |+| `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN` | `750` | clientNotifications invocations per minute |+| `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN` | `600` | promptBridge invocations per minute |+| `CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN` | `3000` | authenticated shared SDK request bucket per minute |+| `CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN` | `600` | anonymous or invalid-credential source-IP pre-auth checks per minute |+| `CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN` | `3000` | verified Host principal pre-auth checks per minute |The defaults above equal the code defaults in `control-api/src/config.ts` andare registered in both `deploy/base/control-plane/configmaps.yaml` and thediff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 2b9c3013c..c6c0eb7d4 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex c43d9d0b3..1086afb5c 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.405",+ "version": "0.1.406","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/types.ts b/workflow-recipes/src/types.tsindex c31b83287..ddd7865d0 100644--- a/workflow-recipes/src/types.ts+++ b/workflow-recipes/src/types.ts@@ -515,7 +515,7 @@ export interface PluginWorkloadSdkSpec {maxRequestContentBytes?: number/** Max parallel promptBridge calls per workload; default 5. */maxConcurrentInvocations?: number- /** Rate limit per workload; platform default 120 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348). */+ /** Rate limit per workload; platform default 600 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348). */maxInvocationsPerMinute?: number/** How long invocation results are queryable; default 24. */resultRetentionHours?: number@@ -530,7 +530,7 @@ export interface PluginWorkloadSdkSpec {allowedUserRefs?: boolean/** @deprecated Accepted for compatibility; no longer enforced (issue #348). Platform per-minute rate limits apply. */maxNotificationsPerRun?: number- /** Rate limit per workload; platform default 150 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348). */+ /** Rate limit per workload; platform default 750 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348). */maxNotificationsPerMinute?: number/** Default 256. */maxTitleBytes?: number