Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): coordinate administrative quota enforcement

Alfredo Lopezb4f73fcf2485
diff --git a/charts/clerum-crds/crds/workflowrecipe.yaml b/charts/clerum-crds/crds/workflowrecipe.yaml
index b8449c761..a27c91bdf 100644
--- a/charts/clerum-crds/crds/workflowrecipe.yaml
+++ b/charts/clerum-crds/crds/workflowrecipe.yaml
@@ -1795,7 +1795,7 @@ spec:
maxInvocationsPerMinute:
type: integer
minimum: 1
- description: Rate limit per workload; platform default 120 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348).
+ description: Rate limit per workload; platform default 600 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348).
resultRetentionHours:
type: integer
minimum: 1
@@ -1842,7 +1842,7 @@ spec:
maxNotificationsPerMinute:
type: integer
minimum: 1
- description: Rate limit per workload; platform default 150 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348).
+ description: Rate limit per workload; platform default 750 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348).
maxTitleBytes:
type: integer
minimum: 1
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 6ad274302..6198436cc 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.707",
+ "version": "0.1.708",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.707",
+ "version": "0.1.708",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 32cb8e0a1..359b6ec2e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.707",
+ "version": "0.1.708",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index 4cda6e62b..f3af91b86 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -211,6 +211,27 @@ type Config = {
adminSubscriptionReadPerMin: number
adminSubscriptionWritePerMin: number
subscriptionOAuthCallbackPerMin: number
+ adminWorkflowReadPerMin: number
+ adminWorkflowGrantReadPerMin: number
+ adminWorkflowGrantWritePerMin: number
+ adminWorkflowTriggerPerMin: number
+ adminOutputsReadPerMin: number
+ llmProviderAttemptAuthorizePerMin: number
+ llmProviderAttemptAuthorizeAnonymousIpPerMin: number
+ adminRegistryKeysPerMin: number
+ adminRegistryGrantsPerMin: number
+ adminRegistryConnectStatusPerMin: number
+ adminRegistryConnectRequestPerMin: number
+ adminRegistryConnectRecoveryPerMin: number
+ adminConnectorDeleteEdgePerMin: number
+ adminConnectorDeletePerMin: number
+ pluginSdkAdminRlPerMin: number
+ pluginSdkInternalRlPerMin: number
+ pluginSdkCredentialRlPerMin: number
+ pluginSdkAuthenticatedPreauthRlPerMin: number
+ adminGfsGrantsPerMin: number
+ adminGfsSharesPerMin: number
+ adminGfsLegacyGrantReportPerMin: number
approvalRlRefreshPerMin: number
approvalRlExternalPerMin: number
approvalRlExternalEdgePerMin: number
@@ -1016,8 +1037,71 @@ export const config: Config = {
'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',
100
),
- approvalRlRequestPerMin: Number(process.env.APPROVAL_RL_REQUEST_PER_MIN || 120),
- approvalRlRefreshPerMin: Number(process.env.APPROVAL_RL_REFRESH_PER_MIN || 20),
+ adminWorkflowReadPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN', 300),
+ adminWorkflowGrantReadPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN',
+ 300
+ ),
+ adminWorkflowGrantWritePerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN',
+ 100
+ ),
+ adminWorkflowTriggerPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN',
+ 50
+ ),
+ adminOutputsReadPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN', 150),
+ llmProviderAttemptAuthorizePerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN',
+ 300
+ ),
+ llmProviderAttemptAuthorizeAnonymousIpPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN',
+ 60
+ ),
+ adminRegistryKeysPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN', 150),
+ adminRegistryGrantsPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN',
+ 150
+ ),
+ adminRegistryConnectStatusPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN',
+ 150
+ ),
+ adminRegistryConnectRequestPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN',
+ 15
+ ),
+ adminRegistryConnectRecoveryPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN',
+ 50
+ ),
+ adminConnectorDeleteEdgePerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN',
+ 300
+ ),
+ adminConnectorDeletePerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN',
+ 150
+ ),
+ pluginSdkAdminRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN', 600),
+ pluginSdkInternalRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN', 600),
+ pluginSdkCredentialRlPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',
+ 1800
+ ),
+ pluginSdkAuthenticatedPreauthRlPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',
+ 6000
+ ),
+ adminGfsGrantsPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN', 150),
+ adminGfsSharesPerMin: positiveIntegerFromEnv('CONTROL_API_ADMIN_GFS_SHARES_PER_MIN', 150),
+ adminGfsLegacyGrantReportPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN',
+ 150
+ ),
+ approvalRlRequestPerMin: positiveIntegerFromEnv('APPROVAL_RL_REQUEST_PER_MIN', 600),
+ approvalRlRefreshPerMin: positiveIntegerFromEnv('APPROVAL_RL_REFRESH_PER_MIN', 100),
approvalRlExternalPerMin: externalRateLimitConfig.operation,
approvalRlExternalEdgePerMin: externalRateLimitConfig.session,
// By default the source-IP backstop is wider than the per-session edge bucket
@@ -1025,7 +1109,7 @@ export const config: Config = {
// platform-wide ceiling. Operators may override these independent scopes;
// crossed values are preserved after the boot advisory above.
approvalRlExternalClientIpPerMin: externalRateLimitConfig.clientIp,
- oauthBrokerRlPerMin: Number(process.env.CONTROL_API_OAUTH_BROKER_RL_PER_MIN || 60),
+ oauthBrokerRlPerMin: positiveIntegerFromEnv('CONTROL_API_OAUTH_BROKER_RL_PER_MIN', 300),
// The public control-admin token routes (password reset, invitation, email
// confirmation) have two buckets per route. The first keys on source IP and
// the submitted value (email, login or token prefix): one person retrying.
@@ -1045,15 +1129,15 @@ export const config: Config = {
// would effectively fail open.
pluginSdkNotificationsRlPerMin: positiveIntegerFromEnv(
'CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN',
- 150
+ 750
),
pluginSdkPromptBridgeRlPerMin: positiveIntegerFromEnv(
'CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN',
- 120
+ 600
),
pluginSdkRequestBucketRlPerMin: positiveIntegerFromEnv(
'CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN',
- 600
+ 6000
),
pluginSdkPreauthRlPerMin: positiveIntegerFromEnv('CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN', 600),
// External GFS authority-boundary budgets. The four keyed by source IP or
@@ -1142,7 +1226,7 @@ export const config: Config = {
// the single-hold mechanism volume and covers one held wake plus prewarm
// bursts from a few concurrent devices (4 x 3 = 12) plus a real message.
// Derivation is regression-guarded by test/config.hostWakeRateLimit.test.ts.
- hostWakeRlPerMin: Number(process.env.CONTROL_API_HOST_WAKE_RL_PER_MIN || 30),
+ hostWakeRlPerMin: positiveIntegerFromEnv('CONTROL_API_HOST_WAKE_RL_PER_MIN', 150),
hostWakeCoalesceWindowMs: Number(process.env.CONTROL_API_HOST_WAKE_COALESCE_WINDOW_MS || 2000),
approvalMediumChallengeTtlSec: Number(
process.env.WORKFLOW_APPROVAL_MEDIUM_CHALLENGE_TTL_SEC || 60 * 60
@@ -1238,7 +1322,7 @@ export const config: Config = {
const n = Number(process.env.NOTIFICATION_DESKTOP_GRACE_SECONDS || 90)
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 90
})(),
- approvalRlReissuePerMin: Number(process.env.APPROVAL_RL_REISSUE_PER_MIN || 5),
+ approvalRlReissuePerMin: positiveIntegerFromEnv('APPROVAL_RL_REISSUE_PER_MIN', 25),
// Cleanup of stale token-bucket rows (older than 5 min — the only active
// window). 5 min lines up with the longest bucket window we use.
approvalRlCleanupIntervalMs: Number(process.env.APPROVAL_RL_CLEANUP_INTERVAL_MS || 5 * 60_000),
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 389f549f3..2e7c59399 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -5,6 +5,8 @@ import {
createBoundedPgPoolForConnection,
} from './boundedPgPool.js'
import { config } from './config.js'
+import { rootLogger } from './observability/logger.js'
+import { applyAdminSubscriptionRateLimitNamespace } from './services/adminSubscriptionRateLimitMigration.js'
import { applyCodexCatalogModelsSchema } from './services/codexSubscriptionCatalog.js'
import {
applyCodexChatgptAccountIdSchema,
@@ -6324,6 +6326,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
version: '0124_entity_change_definer_search_path',
apply: applyEntityChangeDefinerSearchPathSchema,
},
+ {
+ version: '0125_admin_subscription_rate_limit_namespace',
+ apply: applyAdminSubscriptionRateLimitNamespace,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
@@ -6675,7 +6681,7 @@ export async function initDb(db: DbConnector = pool): Promise<void> {
try {
await client.query('ROLLBACK')
} catch (rollbackError) {
- console.warn('[ControlAPI] initDb rollback failed:', rollbackError)
+ rootLogger.warn({ err: rollbackError }, 'Control API migration rollback failed')
}
}
throw error
@@ -6684,7 +6690,7 @@ export async function initDb(db: DbConnector = pool): Promise<void> {
try {
await client.query(`SELECT pg_advisory_unlock(${INIT_DB_LOCK_KEY_SQL})`)
} catch (unlockError) {
- console.warn('[ControlAPI] initDb advisory unlock failed:', unlockError)
+ rootLogger.warn({ err: unlockError }, 'Control API migration advisory unlock failed')
}
}
client.release()
diff --git a/control-api/src/middleware/calendarMinuteRateLimitStore.ts b/control-api/src/middleware/calendarMinuteRateLimitStore.ts
new file mode 100644
index 000000000..b4253b762
--- /dev/null
+++ b/control-api/src/middleware/calendarMinuteRateLimitStore.ts
@@ -0,0 +1,38 @@
+import { type ClientRateLimitInfo, MemoryStore } from 'express-rate-limit'
+
+const MINUTE_MS = 60_000
+
+/**
+ * Verified-principal edge counters use the same calendar minute as the PG
+ * ledger. Server-generated source-IP keys retain MemoryStore's first-hit
+ * window so the independent anonymous safeguard keeps its existing contract.
+ */
+export class CalendarMinuteRateLimitStore extends MemoryStore {
+ private scopedKey(key: string, nowMs: number): string {
+ return key.includes(':ip:') ? key : `${Math.floor(nowMs / MINUTE_MS)}:${key}`
+ }
+
+ override async increment(key: string): Promise<ClientRateLimitInfo> {
+ const nowMs = Date.now()
+ const result = await super.increment(this.scopedKey(key, nowMs))
+ return key.includes(':ip:')
+ ? result
+ : { ...result, resetTime: new Date((Math.floor(nowMs / MINUTE_MS) + 1) * MINUTE_MS) }
+ }
+
+ override async get(key: string): Promise<ClientRateLimitInfo | undefined> {
+ const nowMs = Date.now()
+ const result = await super.get(this.scopedKey(key, nowMs))
+ return !result || key.includes(':ip:')
+ ? result
+ : { ...result, resetTime: new Date((Math.floor(nowMs / MINUTE_MS) + 1) * MINUTE_MS) }
+ }
+
+ override async decrement(key: string): Promise<void> {
+ await super.decrement(this.scopedKey(key, Date.now()))
+ }
+
+ override async resetKey(key: string): Promise<void> {
+ await super.resetKey(this.scopedKey(key, Date.now()))
+ }
+}
diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
index 57bd92721..ce9aa0f74 100644
--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
@@ -1,6 +1,14 @@
import type { Request, RequestHandler } from 'express'
+import { ipKeyGenerator, rateLimit } from 'express-rate-limit'
import { config } from '../config.js'
-import { mcpHostRateLimitBucketKey } from '../utils/auth/mcpHostJwtToken.js'
+import { verifyInternalControlJwt } from '../utils/auth/internalControlToken.js'
+import {
+ mcpHostRateLimitBucketKey,
+ mcpHostVerifiedRateLimitPrincipal,
+ verifyMcpHostAccessJwt,
+} from '../utils/auth/mcpHostJwtToken.js'
+import { extractBearerToken } from '../utils/extractBearerToken.js'
+import { CalendarMinuteRateLimitStore } from './calendarMinuteRateLimitStore.js'
import type { UiAuthedRequest } from './controlUIAuth.js'
import { rateLimitMiddleware } from './rateLimitMiddleware.js'
@@ -18,6 +26,102 @@ export function pluginWorkloadSdkCredentialBucketKey(req: Request): string | nul
return mcpHostRateLimitBucketKey('plugin_workload_sdk_credential', req.mcpHostJwt)
}
+const preauthPrincipalCache = new WeakMap<Request, string | null>()
+
+function verifiedPluginSdkPreauthPrincipal(req: Request): string | null {
+ const cached = preauthPrincipalCache.get(req)
+ if (cached !== undefined) return cached
+ const principal =
+ mcpHostVerifiedRateLimitPrincipal(verifyMcpHostAccessJwt(extractBearerToken(req))) || null
+ preauthPrincipalCache.set(req, principal)
+ return principal
+}
+
+/**
+ * Anonymous and invalid credentials stay on the source-IP pre-auth ceiling.
+ * A verified Host principal gets a separate allowance so that ceiling cannot
+ * silently cap authenticated SDK traffic.
+ */
+export function pluginSdkPreauthAssignment(principal: string | null): {
+ limit: number
+ key: string | null
+} {
+ if (principal) {
+ return {
+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,
+ key: `plugin_workload_sdk_preauth:${principal}`,
+ }
+ }
+ return { limit: config.pluginSdkPreauthRlPerMin, key: null }
+}
+
+export function createPluginWorkloadSdkAnonymousPreauthRateLimit(): RequestHandler {
+ return rateLimit({
+ windowMs: 60_000,
+ limit: config.pluginSdkPreauthRlPerMin,
+ skip: req => verifiedPluginSdkPreauthPrincipal(req) !== null,
+ standardHeaders: 'draft-8',
+ legacyHeaders: false,
+ message: { error: 'Too Many Requests', retryable: true },
+ })
+}
+
+export function createPluginWorkloadSdkAuthenticatedPreauthRateLimit(): RequestHandler {
+ return rateLimit({
+ windowMs: 60_000,
+ store: new CalendarMinuteRateLimitStore(),
+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,
+ skipSuccessfulRequests: false,
+ skipFailedRequests: false,
+ skip: req => verifiedPluginSdkPreauthPrincipal(req) === null,
+ keyGenerator: req => {
+ const assignment = pluginSdkPreauthAssignment(verifiedPluginSdkPreauthPrincipal(req))
+ return assignment.key ?? 'plugin_workload_sdk_preauth:unauthenticated'
+ },
+ standardHeaders: 'draft-8',
+ legacyHeaders: false,
+ message: { error: 'Too Many Requests', retryable: true },
+ })
+}
+
+const internalEdgePrincipalCache = new WeakMap<Request, string | null>()
+
+function verifiedInternalSdkEdgePrincipal(req: Request): string | null {
+ const cached = internalEdgePrincipalCache.get(req)
+ if (cached !== undefined) return cached
+ const token = extractBearerToken(req)
+ const claims = token && token.length <= 4096 ? verifyInternalControlJwt(token) : null
+ // Lossless UTF-16 encoding avoids the reserved source-IP delimiter and URI
+ // encoder errors for any string accepted by the existing signed verifier.
+ // This is edge attribution, never route authorization.
+ const principal = claims
+ ? `${claims.iss}:${Buffer.from(claims.sub, 'utf16le').toString('base64url')}`
+ : null
+ internalEdgePrincipalCache.set(req, principal)
+ return principal
+}
+
+/** Protect JWT verification without combining verified internal callers behind one IP. */
+export function createPluginWorkloadSdkInternalEdgeRateLimit(): RequestHandler {
+ return rateLimit({
+ windowMs: 60_000,
+ store: new CalendarMinuteRateLimitStore(),
+ limit: req =>
+ verifiedInternalSdkEdgePrincipal(req) !== null ? config.pluginSdkInternalRlPerMin : 600,
+ keyGenerator: req => {
+ const principal = verifiedInternalSdkEdgePrincipal(req)
+ return principal !== null
+ ? `plugin_workload_sdk_internal_edge:${principal}`
+ : `plugin_workload_sdk_internal_edge:ip:${ipKeyGenerator(req.ip || 'unknown')}`
+ },
+ skipSuccessfulRequests: false,
+ skipFailedRequests: false,
+ standardHeaders: 'draft-8',
+ legacyHeaders: false,
+ message: { error: 'Too Many Requests', retryable: true },
+ })
+}
+
/**
* Distributed recipe-scoped guard for all authenticated SDK gateway routes.
* The pre-auth IP limiter cannot provide caller isolation, while this PG
@@ -40,7 +144,7 @@ export function createPluginWorkloadSdkRequestRateLimit(): RequestHandler {
export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {
return rateLimitMiddleware({
bucketType: 'plugin_workload_sdk_internal',
- maxPerMinute: 120, // not ENV: not part of the plugin abuse surface (issue #348)
+ maxPerMinute: config.pluginSdkInternalRlPerMin,
getBucketKey: req => {
const claims = req.internalControl
if (!claims) return 'plugin_workload_sdk_internal:unauthenticated'
@@ -59,7 +163,7 @@ export function createPluginWorkloadSdkInternalRateLimit(): RequestHandler {
export function createPluginWorkloadSdkAdminRateLimit(): RequestHandler {
return rateLimitMiddleware({
bucketType: 'plugin_workload_sdk_admin',
- maxPerMinute: 120, // not ENV: not part of the plugin abuse surface (issue #348)
+ maxPerMinute: config.pluginSdkAdminRlPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return `plugin_workload_sdk_admin:${sub || 'unauthenticated'}`
diff --git a/control-api/src/middleware/rateLimitMiddleware.ts b/control-api/src/middleware/rateLimitMiddleware.ts
index fca6bc1c3..3b456fd3f 100644
--- a/control-api/src/middleware/rateLimitMiddleware.ts
+++ b/control-api/src/middleware/rateLimitMiddleware.ts
@@ -33,6 +33,25 @@ export type RateLimitEnforcerOptions = {
*/
export type RateLimitEnforcer = (req: Request, res: Response, key: string) => Promise<boolean>
+/** The last enforcing counter owns every advertised quota/header family. */
+function writeRateLimitHeaders(
+ res: Response,
+ limit: number,
+ remaining: number,
+ resetMs: number
+): void {
+ const resetSeconds = Math.max(1, Math.ceil((resetMs - Date.now()) / 1000))
+ res.setHeader('X-RateLimit-Limit', String(limit))
+ res.setHeader('X-RateLimit-Remaining', String(remaining))
+ res.setHeader('X-RateLimit-Reset', String(Math.floor(resetMs / 1000)))
+ res.setHeader('RateLimit', `limit=${limit}, remaining=${remaining}, reset=${resetSeconds}`)
+ res.setHeader('RateLimit-Policy', `${limit};w=60`)
+ // Keep existing draft-6 clients consistent with the ledger as well.
+ res.setHeader('RateLimit-Limit', String(limit))
+ res.setHeader('RateLimit-Remaining', String(remaining))
+ res.setHeader('RateLimit-Reset', String(resetSeconds))
+}
+
/**
* Factory for a per-request rate limit middleware backed by a PG token bucket.
*
@@ -132,6 +151,11 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim
res.removeHeader('X-RateLimit-Limit')
res.removeHeader('X-RateLimit-Remaining')
res.removeHeader('X-RateLimit-Reset')
+ res.removeHeader('RateLimit')
+ res.removeHeader('RateLimit-Policy')
+ res.removeHeader('RateLimit-Limit')
+ res.removeHeader('RateLimit-Remaining')
+ res.removeHeader('RateLimit-Reset')
res.setHeader('Retry-After', String(RATE_LIMIT_BACKEND_RETRY_AFTER_SECONDS))
res.setHeader('Cache-Control', 'no-store')
res.status(503).json({
@@ -148,9 +172,7 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim
): void {
const retryAfterSec = Math.max(1, Math.ceil((denial.resetMs - Date.now()) / 1000))
res.setHeader('Retry-After', String(retryAfterSec))
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', '0')
- res.setHeader('X-RateLimit-Reset', String(Math.floor(denial.resetMs / 1000)))
+ writeRateLimitHeaders(res, opts.maxPerMinute, 0, denial.resetMs)
if (req.log) {
req.log.warn(
{
@@ -167,7 +189,12 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim
'rate limit exceeded'
)
}
- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds: retryAfterSec })
+ res.status(429).json({
+ error: 'Too Many Requests',
+ code: 'rate_limited',
+ message: `This request limit has been reached. Try again in ${retryAfterSec} seconds.`,
+ retryAfterSeconds: retryAfterSec,
+ })
}
return async function enforce(req: Request, res: Response, rawKey: string): Promise<boolean> {
@@ -193,9 +220,12 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim
return false
}
rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'fallback_allowed' }, 1)
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', String(opts.maxPerMinute - decision.totalHits))
- res.setHeader('X-RateLimit-Reset', String(Math.floor(decision.resetMs / 1000)))
+ writeRateLimitHeaders(
+ res,
+ opts.maxPerMinute,
+ opts.maxPerMinute - decision.totalHits,
+ decision.resetMs
+ )
return true
}
if (!result.allowed) {
@@ -205,9 +235,7 @@ export function createRateLimitEnforcer(opts: RateLimitEnforcerOptions): RateLim
}
rateLimitHitsTotal.inc({ bucket_type: opts.bucketType, result: 'allowed' }, 1)
- res.setHeader('X-RateLimit-Limit', String(opts.maxPerMinute))
- res.setHeader('X-RateLimit-Remaining', String(result.remaining))
- res.setHeader('X-RateLimit-Reset', String(Math.floor(result.resetMs / 1000)))
+ writeRateLimitHeaders(res, opts.maxPerMinute, result.remaining, result.resetMs)
return true
}
}
diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.ts
index 06e699ac7..0468e3dbe 100644
--- a/control-api/src/routes/admin/codexSubscription.ts
+++ b/control-api/src/routes/admin/codexSubscription.ts
@@ -83,8 +83,8 @@ import {
} from '../../services/llmAllowedModelsConfigMap.js'
import { K8sConflictError } from '../../services/resourceService.js'
import {
- adminCodexReadRateLimits,
- adminCodexWriteRateLimits,
+ adminSubscriptionReadRateLimits,
+ adminSubscriptionWriteRateLimits,
} from '../workflows/shared/rateLimit.js'
import { createHostValidationDeps, validateHostSpec } from './hostSpecValidation.js'
import {
@@ -819,63 +819,81 @@ export function createAdminCodexSubscriptionRouter(
router.get(
`${BASE}/connection`,
rejectGrokUnkeyed,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
getConnectionHandler
)
router.post(
`${BASE}/browser/start`,
rejectGrokUnkeyed,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
browserStartHandler
)
router.post(
`${BASE}/device/start`,
rejectGrokUnkeyed,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
deviceStartHandler
)
router.get(
`${BASE}/device/poll`,
rejectGrokUnkeyed,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
devicePollHandler
)
- router.post(`${BASE}/refresh`, rejectGrokUnkeyed, ...adminCodexWriteRateLimits(), refreshHandler)
+ router.post(
+ `${BASE}/refresh`,
+ rejectGrokUnkeyed,
+ ...adminSubscriptionWriteRateLimits(),
+ refreshHandler
+ )
router.post(
`${BASE}/catalog/sync`,
rejectGrokUnkeyed,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
catalogSyncHandler
)
- router.post(`${BASE}/revoke`, rejectGrokUnkeyed, ...adminCodexWriteRateLimits(), revokeHandler)
- router.get(`${BASE}/connections/:key`, ...adminCodexReadRateLimits(), getConnectionHandler)
+ router.post(
+ `${BASE}/revoke`,
+ rejectGrokUnkeyed,
+ ...adminSubscriptionWriteRateLimits(),
+ revokeHandler
+ )
+ router.get(`${BASE}/connections/:key`, ...adminSubscriptionReadRateLimits(), getConnectionHandler)
router.post(
`${BASE}/connections/:key/browser/start`,
rejectGrokUnkeyed,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
browserStartHandler
)
router.post(
`${BASE}/connections/:key/device/start`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
deviceStartHandler
)
router.get(
`${BASE}/connections/:key/device/poll`,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
devicePollHandler
)
- router.post(`${BASE}/connections/:key/refresh`, ...adminCodexWriteRateLimits(), refreshHandler)
+ router.post(
+ `${BASE}/connections/:key/refresh`,
+ ...adminSubscriptionWriteRateLimits(),
+ refreshHandler
+ )
router.post(
`${BASE}/connections/:key/catalog/sync`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
catalogSyncHandler
)
- router.post(`${BASE}/connections/:key/revoke`, ...adminCodexWriteRateLimits(), revokeHandler)
+ router.post(
+ `${BASE}/connections/:key/revoke`,
+ ...adminSubscriptionWriteRateLimits(),
+ revokeHandler
+ )
router.get(
`${BASE}/connections`,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
asyncHandler(async (req, res) => {
if (isGrokReq(req)) {
if (!config.grokSubscriptionEnabled) {
@@ -923,7 +941,7 @@ export function createAdminCodexSubscriptionRouter(
router.post(
`${BASE}/connections`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
asyncHandler(async (req, res) => {
if (isGrokReq(req)) {
if (!config.grokSubscriptionEnabled) {
@@ -1002,7 +1020,7 @@ export function createAdminCodexSubscriptionRouter(
router.get(
`${BASE}/connections/:key/models`,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
asyncHandler(async (req, res) => {
if (isGrokReq(req)) {
if (!config.grokSubscriptionEnabled) {
@@ -1036,7 +1054,7 @@ export function createAdminCodexSubscriptionRouter(
router.patch(
`${BASE}/connections/:key`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
asyncHandler(async (req, res) => {
const grok = isGrokReq(req)
if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
@@ -1101,7 +1119,7 @@ export function createAdminCodexSubscriptionRouter(
router.patch(
`${BASE}/connections/:key/models/:model`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
asyncHandler(async (req, res) => {
const grok = isGrokReq(req)
if (grok ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
@@ -1145,7 +1163,7 @@ export function createAdminCodexSubscriptionRouter(
router.get(
`${BASE}/assignable-hosts`,
- ...adminCodexReadRateLimits(),
+ ...adminSubscriptionReadRateLimits(),
asyncHandler(async (req, res) => {
if (isGrokReq(req) ? !config.grokSubscriptionEnabled : !config.codexSubscriptionEnabled) {
res.status(404).json({ error: 'disabled' })
@@ -1260,12 +1278,12 @@ export function createAdminCodexSubscriptionRouter(
router.post(
`${BASE}/connections/:key/hosts/:hostRef/unbind`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
bindUnbindHandler('unbind')
)
router.post(
`${BASE}/connections/:key/hosts/:hostRef/bind`,
- ...adminCodexWriteRateLimits(),
+ ...adminSubscriptionWriteRateLimits(),
bindUnbindHandler('bind')
)
diff --git a/control-api/src/routes/admin/index.ts b/control-api/src/routes/admin/index.ts
index 400960fc7..f14387666 100644
--- a/control-api/src/routes/admin/index.ts
+++ b/control-api/src/routes/admin/index.ts
@@ -23,6 +23,7 @@ import { createAdminRemoteMcpRouter } from './remoteMcp.js'
import { createAdminResourcesRouter } from './resources.js'
import { createAdminSecretsRouter } from './secrets.js'
import { createAdminSharedFilesystemsRouter } from './sharedFilesystems.js'
+import { createAdminSubscriptionCapabilitiesRouter } from './subscriptionCapabilities.js'
import { createAdminTeamsRouter } from './teams.js'
import { createAdminTracingRouter } from './tracing/index.js'
import { createAdminUsageRouter } from './usage.js'
@@ -30,6 +31,7 @@ import { createAdminUsersRouter } from './users.js'
export function createAdminRouter(gateway: K8sGateway): Router {
const router = Router()
+ router.use(createAdminSubscriptionCapabilitiesRouter())
router.use(createAdminHostsOverviewRouter(gateway))
router.use(createAdminControlAdminsRouter())
router.use(createAdminHostArtifactsRouter(gateway))
diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.ts
index 119630ad0..8ec9e9f5f 100644
--- a/control-api/src/routes/admin/registry.ts
+++ b/control-api/src/routes/admin/registry.ts
@@ -4922,7 +4922,7 @@ export function createAdminRegistryRouter(
// ── Org-scoped publish API keys (efrk_) — owner-gated, proxied to the registry ──
const keysRateLimit = rateLimitMiddleware({
bucketType: 'registry_org_keys',
- maxPerMinute: 30,
+ maxPerMinute: config.adminRegistryKeysPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orgkeys:${sub}` : null
@@ -5058,7 +5058,7 @@ export function createAdminRegistryRouter(
// is audit-only (req.adminAuth.sub); it never lands in granted_by_user_id.
const grantsRateLimit = rateLimitMiddleware({
bucketType: 'registry_org_grants',
- maxPerMinute: 30,
+ maxPerMinute: config.adminRegistryGrantsPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `orggrants:${sub}` : null
diff --git a/control-api/src/routes/admin/registryConnect.ts b/control-api/src/routes/admin/registryConnect.ts
index aeef9f68a..720ea1c03 100644
--- a/control-api/src/routes/admin/registryConnect.ts
+++ b/control-api/src/routes/admin/registryConnect.ts
@@ -244,7 +244,7 @@ export function createRegistryConnectRouter(): Router {
// mount, connect panel mount, refresh button).
rateLimitMiddleware({
bucketType: 'registry_connect_status',
- maxPerMinute: 30,
+ maxPerMinute: config.adminRegistryConnectStatusPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_status:${sub}` : null
@@ -350,13 +350,13 @@ export function createRegistryConnectRouter(): Router {
// Per-admin token bucket — registration is a rare, deliberate,
// once-in-a-deployment-lifetime action, and each call consumes one of
// the shared registry's ~5/day per-IP registration attempts (and can
- // create a deployment row there). 3/min still allows a legitimate retry
+ // create a deployment row there). 15/min still allows a legitimate retry
// after a transient failure (e.g. a typo'd org name) while capping how
// much of that daily budget a looped admin can burn through in a single
// minute.
rateLimitMiddleware({
bucketType: 'registry_connect_request',
- maxPerMinute: 3,
+ maxPerMinute: config.adminRegistryConnectRequestPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_request:${sub}` : null
@@ -612,12 +612,12 @@ export function createRegistryConnectRouter(): Router {
// Per-admin token bucket — every call here rotates the deployment's
// one-time claim token at the shared registry (/:id/claim-token), which
// has no rate limiter of its own and emits a
- // deployment.claim_token_reissued audit event on each rotate. 10/min is
+ // deployment.claim_token_reissued audit event on each rotate. 50/min is
// generous for a human-pressed recovery button while bounding registry
// rotate/audit spam from a looped admin.
rateLimitMiddleware({
bucketType: 'registry_connect_recover',
- maxPerMinute: 10,
+ maxPerMinute: config.adminRegistryConnectRecoveryPerMin,
getBucketKey: req => {
const sub = (req as UiAuthedRequest).adminAuth?.sub
return sub ? `registry_connect_recover:${sub}` : null
diff --git a/control-api/src/routes/admin/secrets.ts b/control-api/src/routes/admin/secrets.ts
index da68bc215..7ce1bcecf 100644
--- a/control-api/src/routes/admin/secrets.ts
+++ b/control-api/src/routes/admin/secrets.ts
@@ -58,7 +58,7 @@ const defaultMcpSecretRollbackPermitStore: McpSecretRollbackPermitStore = {
const mcpSecretDeleteEdgeRateLimit = rateLimit({
windowMs: 60_000,
- limit: 60,
+ limit: config.adminConnectorDeleteEdgePerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
// Keyed on `sub`, not `jti`: `signAdminToken` mints a fresh `jti` on every
@@ -76,7 +76,7 @@ const mcpSecretDeleteEdgeRateLimit = rateLimit({
const mcpSecretDeleteRateLimit = rateLimitMiddleware({
bucketType: 'admin_mcp_secret_delete',
- maxPerMinute: 30,
+ maxPerMinute: config.adminConnectorDeletePerMin,
// Same reasoning as mcpSecretDeleteEdgeRateLimit: the stable principal, so
// re-authenticating does not hand the caller a fresh delete budget.
getBucketKey: req => {
@@ -88,7 +88,7 @@ const mcpSecretDeleteRateLimit = rateLimitMiddleware({
// union has no fail-open member, so both modes still enforce.
//
// 'process-memory', not 'closed': this is an admin-authenticated surface that
- // already sits behind mcpSecretDeleteEdgeRateLimit (in-process, 60/min), which
+ // already sits behind the in-process edge limiter, which
// caps it whether or not Postgres can count. 'closed' answers 503, which would
// deny an operator the ability to delete a Secret precisely during a Postgres
// outage — when removing a compromised Secret matters most. The two 'closed'
diff --git a/control-api/src/routes/admin/subscriptionCapabilities.ts b/control-api/src/routes/admin/subscriptionCapabilities.ts
new file mode 100644
index 000000000..03767e55a
--- /dev/null
+++ b/control-api/src/routes/admin/subscriptionCapabilities.ts
@@ -0,0 +1,22 @@
+import { Router } from 'express'
+import { config } from '../../config.js'
+import { adminSubscriptionReadRateLimits } from '../workflows/shared/rateLimit.js'
+
+/** Feature availability only: this never grants account/model execution authority. */
+export function createAdminSubscriptionCapabilitiesRouter(): Router {
+ const router = Router()
+ router.get(
+ '/admin/llm/providers/capabilities',
+ ...adminSubscriptionReadRateLimits(),
+ (_req, res) => {
+ res.setHeader('Cache-Control', 'private, no-store')
+ res.json({
+ providers: {
+ 'codex-subscription': { enabled: config.codexSubscriptionEnabled },
+ 'grok-subscription': { enabled: config.grokSubscriptionEnabled },
+ },
+ })
+ }
+ )
+ return router
+}
diff --git a/control-api/src/routes/auth/codexSubscription.ts b/control-api/src/routes/auth/codexSubscription.ts
index 2f974bca5..75717a13d 100644
--- a/control-api/src/routes/auth/codexSubscription.ts
+++ b/control-api/src/routes/auth/codexSubscription.ts
@@ -22,7 +22,7 @@ import {
resolveCodexCallbackControlUiBaseUrl,
} from '../../services/codexSubscriptionRedirectUri.js'
import { publishAllowedModelsConfigMapAfterGrantChange } from '../../services/llmAllowedModelsConfigMap.js'
-import { codexOAuthCallbackRateLimits } from '../workflows/shared/rateLimit.js'
+import { subscriptionOAuthCallbackRateLimits } from '../workflows/shared/rateLimit.js'
const log = rootLogger.child({ module: 'auth-codex-subscription' })
@@ -65,7 +65,7 @@ export function createAuthCodexSubscriptionRouter(
router.get(
'/auth/codex-subscription/callback',
- ...codexOAuthCallbackRateLimits(),
+ ...subscriptionOAuthCallbackRateLimits(),
asyncHandler(async (req, res) => {
const code = typeof req.query.code === 'string' ? req.query.code : ''
const state = typeof req.query.state === 'string' ? req.query.state : ''
@@ -88,7 +88,10 @@ export function createAuthCodexSubscriptionRouter(
redirectToCodexSurface(res, 'connected')
} catch (err) {
if (err instanceof CodexSubscriptionOAuthError) {
- log.warn({ event: 'codex_oauth_callback_denied', code: err.code }, 'callback denied')
+ log.warn(
+ { event: 'subscription_oauth_callback_denied', code: err.code },
+ 'callback denied'
+ )
if (err.code === 'disabled') {
res.status(404).json({ error: err.code })
return
diff --git a/control-api/src/routes/gfs/grants.ts b/control-api/src/routes/gfs/grants.ts
index 0faa9c705..343d5d073 100644
--- a/control-api/src/routes/gfs/grants.ts
+++ b/control-api/src/routes/gfs/grants.ts
@@ -1,5 +1,6 @@
import type { Request, Router } from 'express'
import { createHash } from 'node:crypto'
+import { config } from '../../config.js'
import { type DbClient, pool, withTransaction } from '../../db.js'
import type { ExternalGfsAuthority } from '../../gfs/externalAuthority.js'
import { isValidHostSubjectId } from '../../gfs/hostSubject.js'
@@ -891,7 +892,7 @@ export function registerGfsGrantRoutes(router: Router): void {
// first, so adminAuth.sub is always present by the time the limiter keys.
const grantsRateLimit = rateLimitMiddleware({
bucketType: 'gfs_grants',
- maxPerMinute: 30,
+ maxPerMinute: config.adminGfsGrantsPerMin,
getBucketKey: req => {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants:${sub}` : null
diff --git a/control-api/src/routes/gfs/legacyStandaloneGrants.ts b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
index 16daaf772..38187cc20 100644
--- a/control-api/src/routes/gfs/legacyStandaloneGrants.ts
+++ b/control-api/src/routes/gfs/legacyStandaloneGrants.ts
@@ -1,4 +1,5 @@
import type { Request, Response, Router } from 'express'
+import { config } from '../../config.js'
import { pool } from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import { requireAuthForControlUI } from '../../middleware/controlUIAuth.js'
@@ -64,7 +65,7 @@ export function registerLegacyStandaloneGrantReportRoute(router: Router): void {
// consuming the grant-mutation budget while still bounding operator polling.
const legacyGrantReportRateLimit = rateLimitMiddleware({
bucketType: 'gfs_grants_legacy_report',
- maxPerMinute: 30,
+ maxPerMinute: config.adminGfsLegacyGrantReportPerMin,
getBucketKey: req => {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsgrants-legacy:${sub}` : null
diff --git a/control-api/src/routes/gfs/shares.ts b/control-api/src/routes/gfs/shares.ts
index 0bdd0c325..73e45d244 100644
--- a/control-api/src/routes/gfs/shares.ts
+++ b/control-api/src/routes/gfs/shares.ts
@@ -1,4 +1,5 @@
import type { Request, Response, Router } from 'express'
+import { config } from '../../config.js'
import { pool, withTransaction } from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import { requireAuthForControlUI } from '../../middleware/controlUIAuth.js'
@@ -40,7 +41,7 @@ export function registerGfsShareRoutes(router: Router): void {
// Per-admin token bucket, same shape as the folder-grant routes above.
const sharesRateLimit = rateLimitMiddleware({
bucketType: 'gfs_shares',
- maxPerMinute: 30,
+ maxPerMinute: config.adminGfsSharesPerMin,
getBucketKey: req => {
const sub = (req as { adminAuth?: { sub?: string } }).adminAuth?.sub
return sub ? `gfsshares:${sub}` : null
diff --git a/control-api/src/routes/internal/pluginWorkloadSdk.ts b/control-api/src/routes/internal/pluginWorkloadSdk.ts
index 0cee68cb6..4d5edee94 100644
--- a/control-api/src/routes/internal/pluginWorkloadSdk.ts
+++ b/control-api/src/routes/internal/pluginWorkloadSdk.ts
@@ -1,8 +1,10 @@
import { Router } from 'express'
-import { rateLimit } from 'express-rate-limit'
import { asyncHandler } from '../../http/asyncHandler.js'
import { requireInternalControlJwt } from '../../middleware/internalControlJwt.js'
-import { createPluginWorkloadSdkInternalRateLimit } from '../../middleware/pluginWorkloadSdkRateLimits.js'
+import {
+ createPluginWorkloadSdkInternalEdgeRateLimit,
+ createPluginWorkloadSdkInternalRateLimit,
+} from '../../middleware/pluginWorkloadSdkRateLimits.js'
import {
type PluginWorkloadSdkRevocationActor,
finalizePluginWorkloadSdkRevocation,
@@ -57,13 +59,7 @@ export function createInternalPluginWorkloadSdkRouter(): Router {
// the prefix so every current/future internal SDK route inherits the guard.
router.use(
'/internal/plugin-workload-sdk',
- rateLimit({
- windowMs: 60_000,
- limit: 600,
- standardHeaders: 'draft-8',
- legacyHeaders: false,
- message: { error: 'Too Many Requests', retryable: true },
- }),
+ createPluginWorkloadSdkInternalEdgeRateLimit(),
requireInternalControlJwt,
createPluginWorkloadSdkInternalRateLimit()
)
diff --git a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
index 931f43628..d85177a35 100644
--- a/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
+++ b/control-api/src/routes/mcp-host/plugin-workload-sdk.routes.ts
@@ -1,11 +1,12 @@
import { type NextFunction, type Request, type Response, Router } from 'express'
-import { rateLimit } from 'express-rate-limit'
import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'
import { config } from '../../config.js'
import { pool } from '../../db.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import { requireMcpHostJwt } from '../../middleware/mcpHostJwtAuth.js'
import {
+ createPluginWorkloadSdkAnonymousPreauthRateLimit,
+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit,
createPluginWorkloadSdkRequestRateLimit,
pluginWorkloadSdkCredentialBucketKey,
} from '../../middleware/pluginWorkloadSdkRateLimits.js'
@@ -80,7 +81,7 @@ export const PLUGIN_WORKLOAD_SDK_PROMPT_BRIDGE_CONTRACT_VERSION = 2
// grant rate/quota remains the product-level invocation limit.
const pluginWorkloadSdkCredentialRateLimit = rateLimitMiddleware({
bucketType: 'plugin_workload_sdk_credential',
- maxPerMinute: 120,
+ maxPerMinute: config.pluginSdkCredentialRlPerMin,
getBucketKey: pluginWorkloadSdkCredentialBucketKey,
onBackendUnavailable: 'process-memory',
})
@@ -626,15 +627,11 @@ export function createMcpHostPluginWorkloadSdkRoutes(): Router {
// recipe-scoped PG limiter (the outer app bounds JSON bodies).
router.use(
'/mcp-host/plugin-workload-sdk',
- rateLimit({
- windowMs: 60_000,
- // ENV-tunable pre-auth flood guard (issue #348):
- // CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN.
- limit: config.pluginSdkPreauthRlPerMin,
- standardHeaders: 'draft-8',
- legacyHeaders: false,
- message: { error: 'Too Many Requests', retryable: true },
- }),
+ // Invalid or missing credentials stay on the anonymous source-IP gate.
+ // Verified Host principals use a separate pre-auth allowance, then the
+ // recipe-scoped request bucket after authentication.
+ createPluginWorkloadSdkAnonymousPreauthRateLimit(),
+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit(),
requireMcpHostJwt,
requirePluginWorkloadSdkScope,
createPluginWorkloadSdkRequestRateLimit()
diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.ts
index f1f2e5edf..d679444a7 100644
--- a/control-api/src/routes/workflows/shared/rateLimit.ts
+++ b/control-api/src/routes/workflows/shared/rateLimit.ts
@@ -1,8 +1,13 @@
-import type { Request, Response } from 'express'
-import { ipKeyGenerator, rateLimit } from 'express-rate-limit'
+import type { Request, RequestHandler, Response } from 'express'
+import { type RateLimitInfo, ipKeyGenerator, rateLimit } from 'express-rate-limit'
import { createHash } from 'node:crypto'
import { config } from '../../../config.js'
-import { rateLimitMiddleware } from '../../../middleware/rateLimitMiddleware.js'
+import { CalendarMinuteRateLimitStore } from '../../../middleware/calendarMinuteRateLimitStore.js'
+import {
+ type RateLimitEnforcerOptions,
+ rateLimitMiddleware,
+} from '../../../middleware/rateLimitMiddleware.js'
+import { rateLimitHitsTotal } from '../../../observability/metrics.js'
import { verifyAdminToken } from '../../../utils/auth/adminAuthToken.js'
import { verifyExternalSessionToken } from '../../../utils/auth/externalSessionAuthToken.js'
import {
@@ -12,12 +17,31 @@ import {
import { CONTROL_UI_ADMIN_SESSION_COOKIE, readCookie } from '../../../utils/auth/sessionCookies.js'
import { extractBearerToken } from '../../../utils/extractBearerToken.js'
-const WORKFLOW_GRANT_READ_PER_MINUTE = 60
-const WORKFLOW_GRANT_WRITE_PER_MINUTE = 20
-const WORKFLOW_ADMIN_READ_PER_MINUTE = 60
-const ADMIN_OUTPUTS_READ_PER_MINUTE = 30
+// Invalid credentials still use the original source-IP abuse budgets. Raised
+// administrative capacity applies only after the existing signature check.
+const UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE = 60
+const UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE = 20
+const UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE = 60
+const UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE = 30
const WORKFLOW_TRIGGER_PER_MINUTE = 10
-const LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE = 60
+const UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE = 30
+const UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE = 20
+const SUBSCRIPTION_CALLBACK_IP_PER_MINUTE = 20
+
+function hasVerifiedAdminCredential(req: Request): boolean {
+ return verifiedAdminRateLimitIdentity(adminWorkflowRateLimitCredential(req)) !== null
+}
+
+/** Reuse the existing enforcer without raising a distinct unauthenticated gate. */
+function withUnverifiedIpBudget(
+ options: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null },
+ unverifiedMaxPerMinute: number,
+ isVerified: (req: Request) => boolean
+): RequestHandler {
+ const verified = rateLimitMiddleware(options)
+ const unverified = rateLimitMiddleware({ ...options, maxPerMinute: unverifiedMaxPerMinute })
+ return (req, res, next) => (isVerified(req) ? verified : unverified)(req, res, next)
+}
/**
* Credential surface matched by requireAdminWorkflowCaller: bearer for automation,
@@ -37,9 +61,12 @@ export function verifiedAdminRateLimitSubject(credential: string | null): string
return claims?.sub || null
}
-function hashedAdminWorkflowCredentialBucket(prefix: string) {
+function hashedAdminWorkflowCredentialBucket(
+ prefix: string,
+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential
+) {
return (req: Request): string | null => {
- const credential = adminWorkflowRateLimitCredential(req)
+ const credential = getCredential(req)
const sessionIdentity = verifiedAdminRateLimitIdentity(credential)
if (sessionIdentity) {
const hash = createHash('sha256').update(sessionIdentity).digest('hex').slice(0, 32)
@@ -65,8 +92,12 @@ export function shouldSkipWorkflowGrantEdgeRateLimit(req: Request): boolean {
* - Verified admin cookie/bearer → per-session bucket (isolates live sessions)
* - Unverified credential → IP bucket (rotation cannot mint fresh identities)
*/
-export function workflowGrantEdgeRateLimitKey(prefix: string, req: Request): string {
- const sessionIdentity = verifiedAdminRateLimitIdentity(adminWorkflowRateLimitCredential(req))
+export function workflowGrantEdgeRateLimitKey(
+ prefix: string,
+ req: Request,
+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential
+): string {
+ const sessionIdentity = verifiedAdminRateLimitIdentity(getCredential(req))
if (sessionIdentity) {
const hash = createHash('sha256').update(sessionIdentity).digest('hex').slice(0, 32)
return `${prefix}:sub:${hash}`
@@ -74,11 +105,11 @@ export function workflowGrantEdgeRateLimitKey(prefix: string, req: Request): str
return `${prefix}:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`
}
-function workflowGrantEdgeRateKey(prefix: string) {
- return (req: Request): string => workflowGrantEdgeRateLimitKey(prefix, req)
+function workflowGrantEdgeRateKey(prefix: string, getCredential: (req: Request) => string | null) {
+ return (req: Request): string => workflowGrantEdgeRateLimitKey(prefix, req, getCredential)
}
-function workflowGrantEdgeRateLimitHandler(_req: Request, res: Response): void {
+function edgeRateLimitDenied(bucketType: string, req: Request, res: Response): void {
const raw = res.getHeader('Retry-After')
const retryAfterSeconds =
typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0
@@ -86,18 +117,44 @@ function workflowGrantEdgeRateLimitHandler(_req: Request, res: Response): void {
: typeof raw === 'string' && /^\d+$/.test(raw)
? Math.max(1, Number(raw))
: 60
- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })
+ const info = (req as Request & { rateLimit?: RateLimitInfo }).rateLimit
+ rateLimitHitsTotal.inc({ bucket_type: bucketType, result: 'denied' }, 1)
+ req.log?.warn(
+ {
+ event: 'rate_limit_denied',
+ bucketType,
+ count: info?.used,
+ maxPerMinute: info?.limit,
+ source: 'edge-memory',
+ method: req.method,
+ route: typeof req.route?.path === 'string' ? req.route.path : undefined,
+ },
+ 'rate limit exceeded'
+ )
+ res.status(429).json({
+ error: 'Too Many Requests',
+ code: 'rate_limited',
+ message: `This request limit has been reached. Try again in ${retryAfterSeconds} seconds.`,
+ retryAfterSeconds,
+ })
}
-function createWorkflowEdgeRateLimit(prefix: string, limit: number) {
+function createWorkflowEdgeRateLimit(
+ prefix: string,
+ limit: number,
+ unverifiedLimit: number,
+ getCredential: (req: Request) => string | null = adminWorkflowRateLimitCredential
+) {
return rateLimit({
windowMs: 60_000,
- limit,
+ store: new CalendarMinuteRateLimitStore(),
+ limit: req =>
+ verifiedAdminRateLimitIdentity(getCredential(req)) !== null ? limit : unverifiedLimit,
standardHeaders: 'draft-7',
legacyHeaders: false,
- skip: shouldSkipWorkflowGrantEdgeRateLimit,
- keyGenerator: workflowGrantEdgeRateKey(prefix),
- handler: workflowGrantEdgeRateLimitHandler,
+ skip: req => getCredential(req) === null,
+ keyGenerator: workflowGrantEdgeRateKey(prefix, getCredential),
+ handler: (req, res) => edgeRateLimitDenied(prefix, req, res),
})
}
@@ -106,11 +163,19 @@ function createWorkflowEdgeRateLimit(prefix: string, limit: number) {
* Routers that own a rate-limit family instantiate the tuple once and reuse it.
*/
export function workflowGrantReadEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('workflow_grants_read_edge', WORKFLOW_GRANT_READ_PER_MINUTE)
+ return createWorkflowEdgeRateLimit(
+ 'workflow_grants_read_edge',
+ config.adminWorkflowGrantReadPerMin,
+ UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE
+ )
}
export function workflowGrantWriteEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('workflow_grants_write_edge', WORKFLOW_GRANT_WRITE_PER_MINUTE)
+ return createWorkflowEdgeRateLimit(
+ 'workflow_grants_write_edge',
+ config.adminWorkflowGrantWritePerMin,
+ UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE
+ )
}
export function workflowGrantReadRateLimits() {
@@ -122,11 +187,19 @@ export function workflowGrantWriteRateLimits() {
}
function workflowAdminReadEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('workflow_admin_read_edge', WORKFLOW_ADMIN_READ_PER_MINUTE)
+ return createWorkflowEdgeRateLimit(
+ 'workflow_admin_read_edge',
+ config.adminWorkflowReadPerMin,
+ UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE
+ )
}
function adminOutputsReadEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('admin_outputs_read_edge', ADMIN_OUTPUTS_READ_PER_MINUTE)
+ return createWorkflowEdgeRateLimit(
+ 'admin_outputs_read_edge',
+ config.adminOutputsReadPerMin,
+ UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE
+ )
}
export function workflowAdminReadRateLimits() {
@@ -137,65 +210,108 @@ export function adminOutputsReadRateLimits() {
return [adminOutputsReadEdgeRateLimit(), adminOutputsReadRateLimit()] as const
}
-function adminCodexReadEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('admin_codex_read_edge', config.adminSubscriptionReadPerMin)
+/** Subscription routes follow their cookie-only parent authentication. */
+function adminSubscriptionCredential(req: Request): string | null {
+ return readCookie(req, CONTROL_UI_ADMIN_SESSION_COOKIE) || null
}
-function adminCodexWriteEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('admin_codex_write_edge', config.adminSubscriptionWritePerMin)
+function hasVerifiedSubscriptionCredential(req: Request): boolean {
+ return verifiedAdminRateLimitIdentity(adminSubscriptionCredential(req)) !== null
}
-function adminCodexReadRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'admin_codex_read',
- maxPerMinute: config.adminSubscriptionReadPerMin,
- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),
- onBackendUnavailable: 'process-memory',
- })
+function adminSubscriptionReadEdgeRateLimit() {
+ return createWorkflowEdgeRateLimit(
+ 'admin_subscription_read_edge',
+ config.adminSubscriptionReadPerMin,
+ UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE,
+ adminSubscriptionCredential
+ )
}
-function adminCodexWriteRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'admin_codex_write',
- maxPerMinute: config.adminSubscriptionWritePerMin,
- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),
- onBackendUnavailable: 'process-memory',
- })
+function adminSubscriptionWriteEdgeRateLimit() {
+ return createWorkflowEdgeRateLimit(
+ 'admin_subscription_write_edge',
+ config.adminSubscriptionWritePerMin,
+ UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE,
+ adminSubscriptionCredential
+ )
}
-export function adminCodexReadRateLimits() {
- return [adminCodexReadEdgeRateLimit(), adminCodexReadRateLimit()] as const
+function adminSubscriptionReadRateLimit() {
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'admin_subscription_read',
+ maxPerMinute: config.adminSubscriptionReadPerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket(
+ 'admin_subscription_read',
+ adminSubscriptionCredential
+ ),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_ADMIN_SUBSCRIPTION_READ_PER_MINUTE,
+ hasVerifiedSubscriptionCredential
+ )
}
-export function adminCodexWriteRateLimits() {
- return [adminCodexWriteEdgeRateLimit(), adminCodexWriteRateLimit()] as const
+function adminSubscriptionWriteRateLimit() {
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'admin_subscription_write',
+ maxPerMinute: config.adminSubscriptionWritePerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket(
+ 'admin_subscription_write',
+ adminSubscriptionCredential
+ ),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_ADMIN_SUBSCRIPTION_WRITE_PER_MINUTE,
+ hasVerifiedSubscriptionCredential
+ )
}
-function codexOAuthCallbackBucketKey(req: Request): string {
+export function adminSubscriptionReadRateLimits() {
+ return [adminSubscriptionReadEdgeRateLimit(), adminSubscriptionReadRateLimit()] as const
+}
+
+export function adminSubscriptionWriteRateLimits() {
+ return [adminSubscriptionWriteEdgeRateLimit(), adminSubscriptionWriteRateLimit()] as const
+}
+
+function subscriptionOAuthCallbackBucketKey(req: Request): string {
const state = typeof req.query.state === 'string' ? req.query.state.trim() : ''
if (state) {
const hash = createHash('sha256').update(state).digest('hex').slice(0, 32)
- return `codex_oauth_callback:state:${hash}`
+ return `subscription_oauth_callback:state:${hash}`
}
- return `codex_oauth_callback:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`
+ return `subscription_oauth_callback:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`
}
-export function codexOAuthCallbackRateLimits() {
+export function subscriptionOAuthCallbackRateLimits() {
+ const hasState = (req: Request) =>
+ typeof req.query.state === 'string' && req.query.state.trim().length > 0
return [
rateLimit({
windowMs: 60_000,
- limit: config.subscriptionOAuthCallbackPerMin,
+ store: new CalendarMinuteRateLimitStore(),
+ limit: req =>
+ hasState(req)
+ ? config.subscriptionOAuthCallbackPerMin
+ : SUBSCRIPTION_CALLBACK_IP_PER_MINUTE,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: codexOAuthCallbackBucketKey,
- handler: workflowGrantEdgeRateLimitHandler,
- }),
- rateLimitMiddleware({
- bucketType: 'codex_oauth_callback',
- maxPerMinute: config.subscriptionOAuthCallbackPerMin,
- getBucketKey: codexOAuthCallbackBucketKey,
- onBackendUnavailable: 'process-memory',
+ keyGenerator: subscriptionOAuthCallbackBucketKey,
+ handler: (req, res) => edgeRateLimitDenied('subscription_oauth_callback_edge', req, res),
}),
+ withUnverifiedIpBudget(
+ {
+ bucketType: 'subscription_oauth_callback',
+ maxPerMinute: config.subscriptionOAuthCallbackPerMin,
+ getBucketKey: subscriptionOAuthCallbackBucketKey,
+ onBackendUnavailable: 'process-memory',
+ },
+ SUBSCRIPTION_CALLBACK_IP_PER_MINUTE,
+ hasState
+ ),
] as const
}
@@ -205,7 +321,7 @@ export function codexOAuthCallbackRateLimits() {
* 1st-party hosts which share that sentinel and must key by `hostRefs[0]`
* - Missing or unverified bearer → client IP (rotation cannot mint buckets)
*/
-export function mcpHostAttemptRateLimitKey(req: Request): string {
+function mcpHostAttemptRateLimitPrincipal(req: Request): string | null {
const attached = mcpHostVerifiedRateLimitPrincipal(req.mcpHostJwt)
// Always verify. Gating on bearer truthiness is a user-controlled skip of
// the security check (CodeQL js/user-controlled-bypass). Empty or forged
@@ -213,27 +329,40 @@ export function mcpHostAttemptRateLimitKey(req: Request): string {
const verified = mcpHostVerifiedRateLimitPrincipal(
verifyMcpHostAccessJwt(extractBearerToken(req))
)
- const principal = attached ?? verified
+ return attached ?? verified
+}
+
+export function mcpHostAttemptRateLimitKey(req: Request): string {
+ const principal = mcpHostAttemptRateLimitPrincipal(req)
if (principal) return `llm_provider_attempt:${principal}`
return `llm_provider_attempt:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`
}
export function llmProviderAttemptAuthorizeRateLimits() {
+ const hasPrincipal = (req: Request) => mcpHostAttemptRateLimitPrincipal(req) !== null
return [
rateLimit({
windowMs: 60_000,
- limit: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,
+ store: new CalendarMinuteRateLimitStore(),
+ limit: req =>
+ hasPrincipal(req)
+ ? config.llmProviderAttemptAuthorizePerMin
+ : config.llmProviderAttemptAuthorizeAnonymousIpPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: mcpHostAttemptRateLimitKey,
- handler: workflowGrantEdgeRateLimitHandler,
- }),
- rateLimitMiddleware({
- bucketType: 'llm_provider_attempt_authorize',
- maxPerMinute: LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE,
- getBucketKey: mcpHostAttemptRateLimitKey,
- onBackendUnavailable: 'process-memory',
+ handler: (req, res) => edgeRateLimitDenied('llm_provider_attempt_authorize_edge', req, res),
}),
+ withUnverifiedIpBudget(
+ {
+ bucketType: 'llm_provider_attempt_authorize',
+ maxPerMinute: config.llmProviderAttemptAuthorizePerMin,
+ getBucketKey: mcpHostAttemptRateLimitKey,
+ onBackendUnavailable: 'process-memory',
+ },
+ config.llmProviderAttemptAuthorizeAnonymousIpPerMin,
+ hasPrincipal
+ ),
] as const
}
@@ -249,12 +378,16 @@ function adminWorkflowTriggerRateLimitKey(req: Request): string | null {
}
export function adminWorkflowTriggerRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'workflow_trigger',
- maxPerMinute: WORKFLOW_TRIGGER_PER_MINUTE,
- getBucketKey: adminWorkflowTriggerRateLimitKey,
- onBackendUnavailable: 'process-memory',
- })
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'workflow_trigger',
+ maxPerMinute: config.adminWorkflowTriggerPerMin,
+ getBucketKey: adminWorkflowTriggerRateLimitKey,
+ onBackendUnavailable: 'process-memory',
+ },
+ WORKFLOW_TRIGGER_PER_MINUTE,
+ hasVerifiedAdminCredential
+ )
}
function hashedWorkflowTriggerBucket(credential: string): string {
@@ -326,39 +459,55 @@ export function mcpHostWorkflowTriggerRateLimit() {
}
function workflowAdminReadRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'workflow_admin_read',
- maxPerMinute: WORKFLOW_ADMIN_READ_PER_MINUTE,
- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),
- onBackendUnavailable: 'process-memory',
- })
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'workflow_admin_read',
+ maxPerMinute: config.adminWorkflowReadPerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_admin_read'),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE,
+ hasVerifiedAdminCredential
+ )
}
function adminOutputsReadRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'admin_outputs_read',
- maxPerMinute: ADMIN_OUTPUTS_READ_PER_MINUTE,
- getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),
- onBackendUnavailable: 'process-memory',
- })
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'admin_outputs_read',
+ maxPerMinute: config.adminOutputsReadPerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket('admin_outputs_read'),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_ADMIN_OUTPUTS_READ_PER_MINUTE,
+ hasVerifiedAdminCredential
+ )
}
export function workflowGrantReadRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'workflow_grants_read',
- maxPerMinute: WORKFLOW_GRANT_READ_PER_MINUTE,
- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),
- onBackendUnavailable: 'process-memory',
- })
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'workflow_grants_read',
+ maxPerMinute: config.adminWorkflowGrantReadPerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_read'),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE,
+ hasVerifiedAdminCredential
+ )
}
export function workflowGrantWriteRateLimit() {
- return rateLimitMiddleware({
- bucketType: 'workflow_grants_write',
- maxPerMinute: WORKFLOW_GRANT_WRITE_PER_MINUTE,
- getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),
- onBackendUnavailable: 'process-memory',
- })
+ return withUnverifiedIpBudget(
+ {
+ bucketType: 'workflow_grants_write',
+ maxPerMinute: config.adminWorkflowGrantWritePerMin,
+ getBucketKey: hashedAdminWorkflowCredentialBucket('workflow_grants_write'),
+ onBackendUnavailable: 'process-memory',
+ },
+ UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE,
+ hasVerifiedAdminCredential
+ )
}
/** Keep distinct signed sessions in distinct pre-auth quota buckets. */
diff --git a/control-api/src/services/adminSubscriptionRateLimitMigration.ts b/control-api/src/services/adminSubscriptionRateLimitMigration.ts
new file mode 100644
index 000000000..8318ebd0b
--- /dev/null
+++ b/control-api/src/services/adminSubscriptionRateLimitMigration.ts
@@ -0,0 +1,80 @@
+import type { DbClient } from '../db.js'
+
+const NAMESPACE_RENAMES = [
+ ['admin_codex_read:', 'admin_subscription_read:'],
+ ['admin_codex_write:', 'admin_subscription_write:'],
+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],
+] as const
+
+/** Existing opaque long-key digests cannot be backfilled from their hashes. */
+export function legacyAdministrativeRateLimitHashInput(key: string): string {
+ for (const [legacy, current] of NAMESPACE_RENAMES) {
+ if (key.startsWith(current)) return legacy + key.slice(current.length)
+ }
+ return key
+}
+
+/**
+ * Preserve active quota accounting across the provider-neutral rename.
+ * initDb holds a transaction and migration advisory lock. The table lock
+ * serializes the backfill with increments; the invoker trigger also routes
+ * older binaries to the same rows during rolling updates and rollback.
+ * Legacy provider spellings are intentionally confined to this compatibility
+ * boundary. Retire the trigger only after older writers/rollback are retired.
+ */
+export async function applyAdminSubscriptionRateLimitNamespace(db: DbClient): Promise<void> {
+ const cases = NAMESPACE_RENAMES.map(
+ ([legacy, current], index) => `
+ ${index === 0 ? 'IF' : 'ELSIF'} left(input_key, ${legacy.length}) = '${legacy}' THEN
+ canonical_key := '${current}' || substring(input_key FROM ${legacy.length + 1});
+ ELSIF left(input_key, ${current.length}) = '${current}' THEN
+ legacy_input := '${legacy}' || substring(input_key FROM ${current.length + 1});
+ `
+ ).join('\n')
+ await db.query(`
+ LOCK TABLE rate_limit_buckets IN ACCESS EXCLUSIVE MODE;
+
+ CREATE OR REPLACE FUNCTION public.canonical_admin_subscription_bucket_key(input_key TEXT)
+ RETURNS TEXT LANGUAGE plpgsql IMMUTABLE STRICT SECURITY INVOKER
+ SET search_path = pg_catalog AS $$
+ DECLARE
+ canonical_key TEXT := input_key;
+ legacy_input TEXT := input_key;
+ BEGIN
+ ${cases}
+ ELSE
+ RETURN input_key;
+ END IF;
+ IF octet_length(canonical_key) > 512 THEN
+ RETURN 'sha256-long-key:' || encode(sha256(convert_to(legacy_input, 'UTF8')), 'hex');
+ END IF;
+ RETURN canonical_key;
+ END;
+ $$;
+
+ INSERT INTO rate_limit_buckets (bucket_key, window_start_ms, count)
+ SELECT public.canonical_admin_subscription_bucket_key(bucket_key),
+ window_start_ms, SUM(count)::INTEGER
+ FROM rate_limit_buckets
+ WHERE public.canonical_admin_subscription_bucket_key(bucket_key) <> bucket_key
+ GROUP BY public.canonical_admin_subscription_bucket_key(bucket_key), window_start_ms
+ ON CONFLICT (bucket_key, window_start_ms)
+ DO UPDATE SET count = rate_limit_buckets.count + EXCLUDED.count;
+ DELETE FROM rate_limit_buckets
+ WHERE public.canonical_admin_subscription_bucket_key(bucket_key) <> bucket_key;
+
+ CREATE OR REPLACE FUNCTION public.normalize_admin_subscription_bucket_key()
+ RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER
+ SET search_path = pg_catalog AS $$
+ BEGIN
+ NEW.bucket_key := public.canonical_admin_subscription_bucket_key(NEW.bucket_key);
+ RETURN NEW;
+ END;
+ $$;
+
+ DROP TRIGGER IF EXISTS normalize_admin_subscription_bucket_key ON rate_limit_buckets;
+ CREATE TRIGGER normalize_admin_subscription_bucket_key
+ BEFORE INSERT OR UPDATE ON rate_limit_buckets
+ FOR EACH ROW EXECUTE FUNCTION public.normalize_admin_subscription_bucket_key();
+ `)
+}
diff --git a/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts b/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts
index 2f74147d5..adc3c5df8 100644
--- a/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts
+++ b/control-api/src/services/pluginWorkloadSdkQuotaTracker.ts
@@ -13,7 +13,7 @@ import {
// override still wins (decision §3.1) — it is a per-recipe fairness knob, NOT
// the platform DoS ceiling, so it is intentionally NOT clamped to the default.
// A raised override stays bounded by the grant-independent request bucket
-// (CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN, default 600/min) + pre-auth
+// (CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN, default 3000/min) + pre-auth
// limits (pluginWorkloadSdkRateLimits.ts / plugin-workload-sdk.routes.ts),
// which are the actual anti-DoS backstop.
//
diff --git a/control-api/src/services/rateLimitBucketKey.ts b/control-api/src/services/rateLimitBucketKey.ts
index 847a0daf5..53953e170 100644
--- a/control-api/src/services/rateLimitBucketKey.ts
+++ b/control-api/src/services/rateLimitBucketKey.ts
@@ -1,4 +1,5 @@
import { createHash } from 'node:crypto'
+import { legacyAdministrativeRateLimitHashInput } from './adminSubscriptionRateLimitMigration.js'
/** Longest bucket key, in UTF-8 bytes, that the rate limiters store as given. */
export const MAX_RATE_LIMIT_BUCKET_KEY_BYTES = 512
@@ -20,5 +21,8 @@ const LONG_KEY_PREFIX = 'sha256-long-key:'
*/
export function boundedBucketKey(key: string): string {
if (Buffer.byteLength(key, 'utf8') <= MAX_RATE_LIMIT_BUCKET_KEY_BYTES) return key
- return `${LONG_KEY_PREFIX}${createHash('sha256').update(key).digest('hex')}`
+ // Preserve pre-rename digests: the database cannot recover a namespace from
+ // an existing hash. Plain renamed keys are handled by the migration trigger.
+ const hashInput = legacyAdministrativeRateLimitHashInput(key)
+ return `${LONG_KEY_PREFIX}${createHash('sha256').update(hashInput).digest('hex')}`
}
diff --git a/control-api/test/app.test.ts b/control-api/test/app.test.ts
index 9cdbd82c8..22fb8d2f9 100644
--- a/control-api/test/app.test.ts
+++ b/control-api/test/app.test.ts
@@ -17,6 +17,15 @@ function signHccInternalControl(): string {
}
describe('app router wiring', () => {
+ it('requires the real admin session guard for subscription capability discovery', async () => {
+ const app = createApp(new MockGateway('mcp-server') as never)
+ await request(app).get('/api/v1/admin/llm/providers/capabilities').expect(401)
+ await request(app)
+ .get('/api/v1/admin/llm/providers/capabilities')
+ .set('Cookie', 'control_ui_admin_session=unverified-fixture')
+ .expect(401)
+ })
+
afterEach(() => {
vi.restoreAllMocks()
})
diff --git a/control-api/test/config.adminRateLimits.test.ts b/control-api/test/config.adminRateLimits.test.ts
index eb8a3caed..e4c1f4dba 100644
--- a/control-api/test/config.adminRateLimits.test.ts
+++ b/control-api/test/config.adminRateLimits.test.ts
@@ -54,3 +54,78 @@ describe('administrative subscription capacity', () => {
)
}
})
+
+const OPERATING_BUDGETS = [
+ ['CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN', 'adminWorkflowReadPerMin', 300],
+ ['CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN', 'adminWorkflowGrantReadPerMin', 300],
+ ['CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN', 'adminWorkflowGrantWritePerMin', 100],
+ ['CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN', 'adminWorkflowTriggerPerMin', 50],
+ ['CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN', 'adminOutputsReadPerMin', 150],
+ ['CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN', 'llmProviderAttemptAuthorizePerMin', 300],
+ [
+ 'CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN',
+ 'llmProviderAttemptAuthorizeAnonymousIpPerMin',
+ 60,
+ ],
+ ['CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN', 'adminRegistryKeysPerMin', 150],
+ ['CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN', 'adminRegistryGrantsPerMin', 150],
+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN', 'adminRegistryConnectStatusPerMin', 150],
+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN', 'adminRegistryConnectRequestPerMin', 15],
+ ['CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN', 'adminRegistryConnectRecoveryPerMin', 50],
+ ['CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN', 'adminConnectorDeleteEdgePerMin', 300],
+ ['CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN', 'adminConnectorDeletePerMin', 150],
+ ['CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN', 'adminGfsGrantsPerMin', 150],
+ ['CONTROL_API_ADMIN_GFS_SHARES_PER_MIN', 'adminGfsSharesPerMin', 150],
+ ['CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN', 'adminGfsLegacyGrantReportPerMin', 150],
+ ['APPROVAL_RL_REFRESH_PER_MIN', 'approvalRlRefreshPerMin', 100],
+ ['APPROVAL_RL_REISSUE_PER_MIN', 'approvalRlReissuePerMin', 25],
+ ['CONTROL_API_OAUTH_BROKER_RL_PER_MIN', 'oauthBrokerRlPerMin', 300],
+] as const
+
+async function readOperatingBudgets(
+ overrides: Partial<Record<(typeof OPERATING_BUDGETS)[number][0], string>> = {}
+) {
+ for (const [key] of OPERATING_BUDGETS) vi.stubEnv(key, overrides[key] ?? '')
+ vi.resetModules()
+ const { config } = await import('../src/config.js')
+ return Object.fromEntries(OPERATING_BUDGETS.map(([, field]) => [field, config[field]])) as Record<
+ (typeof OPERATING_BUDGETS)[number][1],
+ number
+ >
+}
+
+describe('similar authenticated administrative budgets', () => {
+ it('defaults each family to at least five times its previous ceiling', async () => {
+ const config = await readOperatingBudgets()
+ for (const [, field, expected] of OPERATING_BUDGETS) {
+ expect(config[field]).toBe(expected)
+ }
+ })
+
+ it('keeps the anonymous provider-attempt allowance separate from the verified budget', async () => {
+ const config = await readOperatingBudgets({
+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN: '900',
+ })
+ expect(config.llmProviderAttemptAuthorizePerMin).toBe(900)
+ expect(config.llmProviderAttemptAuthorizeAnonymousIpPerMin).toBe(60)
+ })
+
+ it('honors an explicit override without changing neighboring families', async () => {
+ const config = await readOperatingBudgets({
+ CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN: '7',
+ CONTROL_API_ADMIN_GFS_SHARES_PER_MIN: '40',
+ })
+ expect(config.adminRegistryConnectRequestPerMin).toBe(7)
+ expect(config.adminGfsSharesPerMin).toBe(40)
+ expect(config.adminRegistryConnectRecoveryPerMin).toBe(50)
+ expect(config.adminGfsGrantsPerMin).toBe(150)
+ })
+
+ for (const [key] of OPERATING_BUDGETS) {
+ it(`rejects invalid ${key}`, async () => {
+ await expect(readOperatingBudgets({ [key]: '0' })).rejects.toThrow(
+ `${key} must be a positive integer`
+ )
+ })
+ }
+})
diff --git a/control-api/test/config.approvalRateLimit.test.ts b/control-api/test/config.approvalRateLimit.test.ts
index ec1b3f3ef..d2dcb037e 100644
--- a/control-api/test/config.approvalRateLimit.test.ts
+++ b/control-api/test/config.approvalRateLimit.test.ts
@@ -87,7 +87,7 @@ describe('control-api approval rate limit config', () => {
it('defaults approval request rate limit to multi-agent channel-reader capacity', async () => {
const config = await loadConfigWith({})
- expect(config.approvalRlRequestPerMin).toBe(120)
+ expect(config.approvalRlRequestPerMin).toBe(600)
})
it('accepts the approval request rate limit environment override', async () => {
diff --git a/control-api/test/config.hostWakeRateLimit.test.ts b/control-api/test/config.hostWakeRateLimit.test.ts
index f46f7218b..4908cbb27 100644
--- a/control-api/test/config.hostWakeRateLimit.test.ts
+++ b/control-api/test/config.hostWakeRateLimit.test.ts
@@ -137,10 +137,16 @@ describe('host-wake rate limit budget (cross-service derivation guard)', () => {
)
})
- it('defaults host wake rate limit to the derived 30/min', async () => {
+ it('defaults host wake rate limit to the authenticated 150/min budget', async () => {
const config = await loadConfigWith({})
- expect(config.hostWakeRlPerMin).toBe(30)
+ expect(config.hostWakeRlPerMin).toBe(150)
+ })
+
+ it.each(['0', '-1', '1.5', 'nope'])('rejects invalid host wake budget %s', async value => {
+ await expect(loadConfigWith({ CONTROL_API_HOST_WAKE_RL_PER_MIN: value })).rejects.toThrow(
+ 'CONTROL_API_HOST_WAKE_RL_PER_MIN must be a positive integer'
+ )
})
it('accepts the host wake rate limit environment override', async () => {
diff --git a/control-api/test/config.pluginSdkRateLimits.test.ts b/control-api/test/config.pluginSdkRateLimits.test.ts
index 201f5b9b1..a0305b7c7 100644
--- a/control-api/test/config.pluginSdkRateLimits.test.ts
+++ b/control-api/test/config.pluginSdkRateLimits.test.ts
@@ -11,9 +11,9 @@ import { readFileSync } from 'node:fs'
* now live and this suite is GREEN. Do not weaken these assertions.
*
* Covered:
- * 1. Code defaults 150/120/600/600 (decision Q3: recalibrated from the old
- * hardcoded 120/60; request-bucket and pre-auth unchanged at 600 but
- * moved to ENV).
+ * 1. Code defaults 750/600/6000/600, plus authenticated pre-auth 6000 and
+ * credential operations 1800. The 600 pre-auth value remains the anonymous or
+ * invalid-credential source-IP ceiling.
* 2. Each ENV override honored.
* 3. Invalid values fail loudly at import (positiveIntegerFromEnv).
* 4. Empty string falls back to the code default.
@@ -28,6 +28,10 @@ const RATE_LIMIT_KEYS = [
'CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN',
'CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN',
'CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN',
+ 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',
+ 'CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN',
+ 'CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN',
+ 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',
] as const
type RateLimitKey = (typeof RATE_LIMIT_KEYS)[number]
@@ -37,23 +41,43 @@ const EXPECTED: Array<{ env: RateLimitKey; field: string; defaultValue: number }
{
env: 'CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN',
field: 'pluginSdkNotificationsRlPerMin',
- defaultValue: 150,
+ defaultValue: 750,
},
{
env: 'CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN',
field: 'pluginSdkPromptBridgeRlPerMin',
- defaultValue: 120,
+ defaultValue: 600,
},
{
env: 'CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN',
field: 'pluginSdkRequestBucketRlPerMin',
- defaultValue: 600,
+ defaultValue: 6000,
},
{
env: 'CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN',
field: 'pluginSdkPreauthRlPerMin',
defaultValue: 600,
},
+ {
+ env: 'CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN',
+ field: 'pluginSdkAuthenticatedPreauthRlPerMin',
+ defaultValue: 6000,
+ },
+ {
+ env: 'CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN',
+ field: 'pluginSdkAdminRlPerMin',
+ defaultValue: 600,
+ },
+ {
+ env: 'CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN',
+ field: 'pluginSdkInternalRlPerMin',
+ defaultValue: 600,
+ },
+ {
+ env: 'CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN',
+ field: 'pluginSdkCredentialRlPerMin',
+ defaultValue: 1800,
+ },
]
async function loadConfigWith(overrides: Partial<Record<RateLimitKey, string>>) {
@@ -97,13 +121,17 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {
vi.resetModules()
})
- it('defaults the four platform rate limits to 150/120/600/600', async () => {
+ it('defaults platform rate limits without raising the anonymous pre-auth ceiling', async () => {
const config = await loadConfigWith({})
- expect(config.pluginSdkNotificationsRlPerMin).toBe(150)
- expect(config.pluginSdkPromptBridgeRlPerMin).toBe(120)
- expect(config.pluginSdkRequestBucketRlPerMin).toBe(600)
+ expect(config.pluginSdkNotificationsRlPerMin).toBe(750)
+ expect(config.pluginSdkPromptBridgeRlPerMin).toBe(600)
+ expect(config.pluginSdkRequestBucketRlPerMin).toBe(6000)
expect(config.pluginSdkPreauthRlPerMin).toBe(600)
+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)
+ expect(config.pluginSdkAdminRlPerMin).toBe(600)
+ expect(config.pluginSdkInternalRlPerMin).toBe(600)
+ expect(config.pluginSdkCredentialRlPerMin).toBe(1800)
})
it.each(EXPECTED)('honors the $env override', async ({ env, field }) => {
@@ -124,12 +152,18 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {
CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: '',
CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: '',
CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: '',
+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: '',
+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: '',
+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: '',
+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: '',
})
- expect(config.pluginSdkNotificationsRlPerMin).toBe(150)
- expect(config.pluginSdkPromptBridgeRlPerMin).toBe(120)
- expect(config.pluginSdkRequestBucketRlPerMin).toBe(600)
+ expect(config.pluginSdkNotificationsRlPerMin).toBe(750)
+ expect(config.pluginSdkPromptBridgeRlPerMin).toBe(600)
+ expect(config.pluginSdkRequestBucketRlPerMin).toBe(6000)
expect(config.pluginSdkPreauthRlPerMin).toBe(600)
+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)
+ expect(config.pluginSdkCredentialRlPerMin).toBe(1800)
})
it('registers every key at the code default in the base ConfigMap', async () => {
@@ -168,7 +202,7 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {
/maxInvocationsPerMinute:\s*\n\s*type: integer\s*\n\s*minimum: 1\s*\n\s*description: (.+)/,
'maxInvocationsPerMinute description in charts/clerum-crds/crds/workflowrecipe.yaml'
)
- expect(invocationsDescription).toContain('default 120')
+ expect(invocationsDescription).toContain('default 600')
expect(invocationsDescription).toContain('CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN')
const notificationsDescription = extractOne(
@@ -176,7 +210,7 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {
/maxNotificationsPerMinute:\s*\n\s*type: integer\s*\n\s*minimum: 1\s*\n\s*description: (.+)/,
'maxNotificationsPerMinute description in charts/clerum-crds/crds/workflowrecipe.yaml'
)
- expect(notificationsDescription).toContain('default 150')
+ expect(notificationsDescription).toContain('default 750')
expect(notificationsDescription).toContain('CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN')
})
@@ -195,13 +229,39 @@ describe('plugin SDK platform rate-limit config (issue #348)', () => {
expect(field).toBe('pluginSdkRequestBucketRlPerMin')
})
- it('wires the pre-auth flood-guard limit from config in the SDK routes', () => {
- const source = read('../src/routes/mcp-host/plugin-workload-sdk.routes.ts')
- const field = extractOne(
- source,
- /limit:\s*config\.(pluginSdkPreauthRlPerMin)\b/,
- 'limit wired from config in src/routes/mcp-host/plugin-workload-sdk.routes.ts'
- )
- expect(field).toBe('pluginSdkPreauthRlPerMin')
+ it('keeps anonymous pre-auth separate from the verified principal allowance', () => {
+ const source = read('../src/middleware/pluginWorkloadSdkRateLimits.ts')
+ expect(source).toContain('limit: config.pluginSdkPreauthRlPerMin')
+ expect(source).toContain('limit: config.pluginSdkAuthenticatedPreauthRlPerMin')
+ expect(source).toContain('maxPerMinute: config.pluginSdkAdminRlPerMin')
+ expect(source).toContain('maxPerMinute: config.pluginSdkInternalRlPerMin')
+
+ const routes = read('../src/routes/mcp-host/plugin-workload-sdk.routes.ts')
+ const mounted = routes.slice(routes.indexOf("'/mcp-host/plugin-workload-sdk'"))
+ const anonymous = mounted.indexOf('createPluginWorkloadSdkAnonymousPreauthRateLimit()')
+ const authenticated = mounted.indexOf('createPluginWorkloadSdkAuthenticatedPreauthRateLimit()')
+ const jwt = mounted.indexOf('requireMcpHostJwt,')
+ const bucket = mounted.indexOf('createPluginWorkloadSdkRequestRateLimit()')
+ expect(anonymous).toBeGreaterThan(-1)
+ expect(anonymous).toBeLessThan(authenticated)
+ expect(authenticated).toBeLessThan(jwt)
+ expect(jwt).toBeLessThan(bucket)
+ })
+
+ it('gives a verified principal its own pre-auth allowance', async () => {
+ const config = await loadConfigWith({})
+ const { pluginSdkPreauthAssignment } =
+ await import('../src/middleware/pluginWorkloadSdkRateLimits.js')
+
+ expect(pluginSdkPreauthAssignment(null)).toEqual({
+ limit: config.pluginSdkPreauthRlPerMin,
+ key: null,
+ })
+ expect(pluginSdkPreauthAssignment('recipe:alpha')).toEqual({
+ limit: config.pluginSdkAuthenticatedPreauthRlPerMin,
+ key: 'plugin_workload_sdk_preauth:recipe:alpha',
+ })
+ expect(config.pluginSdkPreauthRlPerMin).toBe(600)
+ expect(config.pluginSdkAuthenticatedPreauthRlPerMin).toBe(6000)
})
})
diff --git a/control-api/test/deploy.channelReaderControlApiBoundary.test.ts b/control-api/test/deploy.channelReaderControlApiBoundary.test.ts
index ec16300ee..ca27f2774 100644
--- a/control-api/test/deploy.channelReaderControlApiBoundary.test.ts
+++ b/control-api/test/deploy.channelReaderControlApiBoundary.test.ts
@@ -128,7 +128,7 @@ describe('channel-reader -> control-api boundary manifests', () => {
)
expect(pollIntervalSeconds).toBe(2)
- expect(approvalRequestsPerMinute).toBe(120)
+ expect(approvalRequestsPerMinute).toBe(600)
expect(approvalRequestsPerMinute).toBeGreaterThanOrEqual((60 / pollIntervalSeconds) * 4)
})
diff --git a/control-api/test/middleware.calendarMinuteRateLimitStore.test.ts b/control-api/test/middleware.calendarMinuteRateLimitStore.test.ts
new file mode 100644
index 000000000..b0d255c62
--- /dev/null
+++ b/control-api/test/middleware.calendarMinuteRateLimitStore.test.ts
@@ -0,0 +1,69 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { Options } from 'express-rate-limit'
+import { CalendarMinuteRateLimitStore } from '../src/middleware/calendarMinuteRateLimitStore.js'
+
+describe('CalendarMinuteRateLimitStore', () => {
+ let store: CalendarMinuteRateLimitStore
+
+ beforeEach(() => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))
+ store = new CalendarMinuteRateLimitStore()
+ store.init({ windowMs: 60_000 } as Options)
+ })
+
+ afterEach(() => {
+ store.shutdown()
+ vi.useRealTimers()
+ })
+
+ it('reports the calendar boundary and restores a verified principal at that boundary after an offset first hit', async () => {
+ const key = 'plugin_workload_sdk_internal_edge:wrc:wrc-provisioner'
+ await expect(store.increment(key)).resolves.toEqual({
+ totalHits: 1,
+ resetTime: new Date('2026-10-02T12:01:00.000Z'),
+ })
+ await store.increment(key)
+ await expect(store.get(key)).resolves.toEqual({
+ totalHits: 2,
+ resetTime: new Date('2026-10-02T12:01:00.000Z'),
+ })
+ await vi.advanceTimersByTimeAsync(42_500)
+ await expect(store.get(key)).resolves.toBeUndefined()
+ await expect(store.increment(key)).resolves.toEqual({
+ totalHits: 1,
+ resetTime: new Date('2026-10-02T12:02:00.000Z'),
+ })
+ })
+
+ it('retains the anonymous source-IP first-hit window across a calendar boundary', async () => {
+ const key = 'plugin_workload_sdk_internal_edge:ip:198.51.100.9'
+ await expect(store.increment(key)).resolves.toEqual({
+ totalHits: 1,
+ resetTime: new Date('2026-10-02T12:01:17.500Z'),
+ })
+ await vi.advanceTimersByTimeAsync(42_500)
+ await expect(store.increment(key)).resolves.toEqual({
+ totalHits: 2,
+ resetTime: new Date('2026-10-02T12:01:17.500Z'),
+ })
+ await vi.advanceTimersByTimeAsync(17_500)
+ await expect(store.increment(key)).resolves.toEqual({
+ totalHits: 1,
+ resetTime: new Date('2026-10-02T12:02:17.500Z'),
+ })
+ })
+
+ it('uses the same scoped key for decrement and reset without affecting another principal', async () => {
+ const first = 'plugin_workload_sdk_preauth:recipe-one'
+ const second = 'plugin_workload_sdk_preauth:recipe-two'
+ await store.increment(first)
+ await store.increment(first)
+ await store.increment(second)
+ await store.decrement(first)
+ await expect(store.get(first)).resolves.toMatchObject({ totalHits: 1 })
+ await store.resetKey(first)
+ await expect(store.get(first)).resolves.toBeUndefined()
+ await expect(store.get(second)).resolves.toMatchObject({ totalHits: 1 })
+ })
+})
diff --git a/control-api/test/middleware.rateLimitMiddleware.test.ts b/control-api/test/middleware.rateLimitMiddleware.test.ts
index d096a78de..c21f17672 100644
--- a/control-api/test/middleware.rateLimitMiddleware.test.ts
+++ b/control-api/test/middleware.rateLimitMiddleware.test.ts
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
-import { MemoryStore } from 'express-rate-limit'
+import { MemoryStore, rateLimit } from 'express-rate-limit'
import { createHash } from 'node:crypto'
import request from 'supertest'
import type { RateLimitCheck } from '../src/services/rateLimiterService.js'
@@ -71,6 +71,66 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
hits.inc.mockReset()
})
+ it.each([1, LIMIT + 1])(
+ 'publishes authoritative ledger headers after a fresh edge (count %i)',
+ async count => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(resetMs - 30_000)
+ try {
+ checkAndIncrement.mockResolvedValue(available(count))
+ const app = express()
+ app.get(
+ '/limited',
+ rateLimit({
+ windowMs: 60_000,
+ limit: 100,
+ standardHeaders: 'draft-7',
+ legacyHeaders: false,
+ }),
+ rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'unit:key',
+ onBackendUnavailable: 'closed',
+ }),
+ (_req, res) => res.sendStatus(204)
+ )
+ const response = await request(app).get('/limited')
+ expect(response.status).toBe(count <= LIMIT ? 204 : 429)
+ const remaining = Math.max(0, LIMIT - count)
+ expect(response.headers.ratelimit).toBe(`limit=${LIMIT}, remaining=${remaining}, reset=30`)
+ expect(response.headers['ratelimit-policy']).toBe(`${LIMIT};w=60`)
+ expect(response.headers['x-ratelimit-limit']).toBe(String(LIMIT))
+ expect(response.headers['x-ratelimit-remaining']).toBe(String(remaining))
+ if (count > LIMIT) expect(response.headers['retry-after']).toBe('30')
+ } finally {
+ vi.useRealTimers()
+ }
+ }
+ )
+
+ it('removes stale edge quota metadata when the authoritative backend cannot count', async () => {
+ checkAndIncrement.mockResolvedValue(unavailable())
+ const app = express()
+ app.get(
+ '/limited',
+ rateLimit({ windowMs: 60_000, limit: 100, standardHeaders: 'draft-7', legacyHeaders: false }),
+ rateLimitMiddleware({
+ bucketType: 'unit_bucket',
+ maxPerMinute: LIMIT,
+ getBucketKey: () => 'unit:key',
+ onBackendUnavailable: 'closed',
+ }),
+ (_req, res) => res.sendStatus(204)
+ )
+ const response = await request(app).get('/limited')
+ expect(response.status).toBe(503)
+ expect(response.headers.ratelimit).toBeUndefined()
+ expect(response.headers['ratelimit-policy']).toBeUndefined()
+ expect(response.headers['x-ratelimit-limit']).toBeUndefined()
+ expect(response.headers['retry-after']).toBe('2')
+ })
+
it("'closed' answers 503 with Retry-After when the backend cannot count the request", async () => {
checkAndIncrement.mockResolvedValueOnce(unavailable())
const { app, handler, warn } = appWith('closed')
@@ -145,7 +205,12 @@ describe('rateLimitMiddleware backend-unavailable policy', () => {
const retryAfterSeconds = Number(denied.headers['retry-after'])
expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
expect(retryAfterSeconds).toBeLessThanOrEqual(60)
- expect(denied.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds })
+ expect(denied.body).toEqual({
+ error: 'Too Many Requests',
+ code: 'rate_limited',
+ message: `This request limit has been reached. Try again in ${retryAfterSeconds} seconds.`,
+ retryAfterSeconds,
+ })
expect(denied.headers['x-ratelimit-limit']).toBe(String(LIMIT))
expect(denied.headers['x-ratelimit-remaining']).toBe('0')
expect(Number(denied.headers['x-ratelimit-reset'])).toBeGreaterThan(Date.now() / 1000)
diff --git a/control-api/test/pluginWorkloadSdkRateLimits.test.ts b/control-api/test/pluginWorkloadSdkRateLimits.test.ts
index bf9b5864f..a693e992b 100644
--- a/control-api/test/pluginWorkloadSdkRateLimits.test.ts
+++ b/control-api/test/pluginWorkloadSdkRateLimits.test.ts
@@ -1,7 +1,11 @@
-import { describe, expect, it } from 'vitest'
-import type express from 'express'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
import { config } from '../src/config.js'
+import { requireMcpHostJwt } from '../src/middleware/mcpHostJwtAuth.js'
import {
+ createPluginWorkloadSdkAnonymousPreauthRateLimit,
+ createPluginWorkloadSdkAuthenticatedPreauthRateLimit,
pluginWorkloadSdkCredentialBucketKey,
pluginWorkloadSdkRequestBucketKey,
} from '../src/middleware/pluginWorkloadSdkRateLimits.js'
@@ -64,3 +68,59 @@ describe('pluginWorkloadSdkRateLimits standalone isolation', () => {
)
})
})
+
+describe('authenticated SDK pre-auth calendar recovery', () => {
+ const originalAuthenticated = config.pluginSdkAuthenticatedPreauthRlPerMin
+ const originalAnonymous = config.pluginSdkPreauthRlPerMin
+
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))
+ config.pluginSdkAuthenticatedPreauthRlPerMin = 2
+ config.pluginSdkPreauthRlPerMin = 2
+ })
+
+ afterEach(() => {
+ config.pluginSdkAuthenticatedPreauthRlPerMin = originalAuthenticated
+ config.pluginSdkPreauthRlPerMin = originalAnonymous
+ vi.useRealTimers()
+ })
+
+ function app() {
+ const instance = express()
+ instance.use(createPluginWorkloadSdkAnonymousPreauthRateLimit())
+ instance.use(createPluginWorkloadSdkAuthenticatedPreauthRateLimit())
+ instance.get('/sdk-gate', requireMcpHostJwt, (_req, res) => res.json({ ok: true }))
+ return instance
+ }
+
+ function signedCaller() {
+ return issueMcpHostAccessJwt('sandbox-recipes', 'calendar-sdk', undefined, {
+ workflowControlScopes: ['plugin-workload-sdk'],
+ }).token
+ }
+
+ it('allows the verified caller at the PG calendar boundary instead of waiting until its first-hit deadline', async () => {
+ const instance = app()
+ const authorization = `Bearer ${signedCaller()}`
+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)
+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)
+ const denied = await request(instance).get('/sdk-gate').set('Authorization', authorization)
+ expect(denied.status).toBe(429)
+ expect(denied.headers['retry-after']).toBe('43')
+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))
+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)
+ })
+
+ it('retains anonymous first-hit denial across that boundary without consuming the verified caller allowance', async () => {
+ const instance = app()
+ await request(instance).get('/sdk-gate').expect(401)
+ await request(instance).get('/sdk-gate').expect(401)
+ await request(instance).get('/sdk-gate').expect(429)
+ const authorization = `Bearer ${signedCaller()}`
+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)
+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))
+ await request(instance).get('/sdk-gate').expect(429)
+ await request(instance).get('/sdk-gate').set('Authorization', authorization).expect(200)
+ })
+})
diff --git a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
index bbcc366de..f17478a2f 100644
--- a/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
+++ b/control-api/test/pluginWorkloadSdkSteplessQuota.realPostgres.integration.test.ts
@@ -332,7 +332,7 @@ describeRealPostgres(
})
it('enforces the per-minute window from the audit trail and frees after 61 seconds (backdated)', async () => {
- // promptBridge at the 120/min platform default: 121 in-window rows deny.
+ // promptBridge at the 600/min platform default: 601 in-window rows deny.
const bridgeRecipe = `stepless-window-bridge-${randomBytes(4).toString('hex')}`
await assertStepless(bridgeRecipe)
await seedPromptBridgeGrant(bridgeRecipe, {})
@@ -342,7 +342,7 @@ describeRealPostgres(
if (!grant) throw new Error(`seeded promptBridge grant not found for ${bridgeRecipe}`)
return grant
})()
- await seedInvocationWindowRows(bridgeRecipe, 'promptBridge', 'window-probe', 121)
+ await seedInvocationWindowRows(bridgeRecipe, 'promptBridge', 'window-probe', 601)
await expect(
quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)
@@ -350,7 +350,7 @@ describeRealPostgres(
ok: false,
error: 'quota_exceeded',
retryable: false,
- message: expect.stringContaining('120/minute'),
+ message: expect.stringContaining('600/minute'),
})
// Backdate the whole burst out of the trailing minute (the window clock is
@@ -365,7 +365,7 @@ describeRealPostgres(
quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)
).resolves.toEqual({ ok: true })
- // clientNotifications mirror at the 150/min platform default, narrowed to
+ // clientNotifications mirror at the 750/min platform default, narrowed to
// the eventType (detail column).
const notifRecipe = `stepless-window-notif-${randomBytes(4).toString('hex')}`
await assertStepless(notifRecipe)
@@ -376,7 +376,7 @@ describeRealPostgres(
if (!grant) throw new Error(`seeded clientNotifications grant not found for ${notifRecipe}`)
return grant
})()
- await seedInvocationWindowRows(notifRecipe, 'clientNotifications', 'lead.followup.due', 151)
+ await seedInvocationWindowRows(notifRecipe, 'clientNotifications', 'lead.followup.due', 751)
await expect(
quotaTracker.checkRateLimit(NS, notifRecipe, 'clientNotifications', notifGrant, {
@@ -386,7 +386,7 @@ describeRealPostgres(
ok: false,
error: 'quota_exceeded',
retryable: false,
- message: expect.stringContaining('150/minute'),
+ message: expect.stringContaining('750/minute'),
})
await db.pool.query(
diff --git a/control-api/test/routes.admin.subscriptionCapabilities.test.ts b/control-api/test/routes.admin.subscriptionCapabilities.test.ts
new file mode 100644
index 000000000..418ecc3e6
--- /dev/null
+++ b/control-api/test/routes.admin.subscriptionCapabilities.test.ts
@@ -0,0 +1,101 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import { config } from '../src/config.js'
+import { requireAuthForControlUI } from '../src/middleware/controlUIAuth.js'
+import { createAdminSubscriptionCapabilitiesRouter } from '../src/routes/admin/subscriptionCapabilities.js'
+import { signAdminToken, verifyAdminToken } from '../src/utils/auth/adminAuthToken.js'
+
+const authenticate = vi.hoisted(() => vi.fn())
+const increment = vi.hoisted(() => vi.fn())
+vi.mock('../src/services/adminSessionAuth.js', () => ({ authenticateAdminSession: authenticate }))
+vi.mock('../src/services/rateLimiterService.js', async importOriginal => ({
+ ...(await importOriginal<typeof import('../src/services/rateLimiterService.js')>()),
+ checkAndIncrement: increment,
+}))
+
+const original = { codex: config.codexSubscriptionEnabled, grok: config.grokSubscriptionEnabled }
+let session: string
+
+function app() {
+ const instance = express()
+ instance.use('/api/v1', requireAuthForControlUI, createAdminSubscriptionCapabilitiesRouter())
+ return instance
+}
+
+beforeEach(() => {
+ // Only live database session state is substituted; quota attribution still
+ // verifies the actual signed browser credential with the production verifier.
+ session = signAdminToken('unit-admin')
+ const claims = verifyAdminToken(session)
+ authenticate.mockImplementation(async value => (value === session ? claims : null))
+ increment.mockResolvedValue({
+ allowed: true,
+ remaining: 149,
+ backendAvailable: true,
+ count: 1,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ })
+})
+
+afterEach(() => {
+ config.codexSubscriptionEnabled = original.codex
+ config.grokSubscriptionEnabled = original.grok
+ vi.clearAllMocks()
+})
+
+describe('administrative subscription capabilities', () => {
+ it.each([
+ [true, true],
+ [true, false],
+ [false, true],
+ [false, false],
+ ])('returns allowlisted integration flags for %s/%s', async (codex, grok) => {
+ config.codexSubscriptionEnabled = codex
+ config.grokSubscriptionEnabled = grok
+ const response = await request(app())
+ .get('/api/v1/admin/llm/providers/capabilities')
+ .set('Cookie', `control_ui_admin_session=${session}`)
+ .expect(200)
+ expect(response.body).toEqual({
+ providers: {
+ 'codex-subscription': { enabled: codex },
+ 'grok-subscription': { enabled: grok },
+ },
+ })
+ expect(response.headers['cache-control']).toBe('private, no-store')
+ expect(response.headers['x-ratelimit-limit']).toBe('150')
+ })
+
+ it('rejects missing or unverified sessions before capability discovery', async () => {
+ await request(app()).get('/api/v1/admin/llm/providers/capabilities').expect(401)
+ await request(app())
+ .get('/api/v1/admin/llm/providers/capabilities')
+ .set('Cookie', 'control_ui_admin_session=unverified-fixture')
+ .expect(401)
+ await request(app())
+ .get('/api/v1/admin/llm/providers/capabilities')
+ .set('Authorization', `Bearer ${session}`)
+ .expect(401)
+ expect(increment).not.toHaveBeenCalled()
+ })
+
+ it('retains the rate limit for authenticated capability reads', async () => {
+ increment.mockResolvedValue({
+ allowed: false,
+ remaining: 0,
+ backendAvailable: true,
+ count: 151,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ })
+ const response = await request(app())
+ .get('/api/v1/admin/llm/providers/capabilities')
+ .set('Cookie', `control_ui_admin_session=${session}`)
+ .expect(429)
+ expect(response.body.code).toBe('rate_limited')
+ expect(response.body.message).toMatch(/Try again/)
+ expect(response.headers['retry-after']).toBeDefined()
+ })
+})
diff --git a/control-api/test/routes.internalPluginWorkloadSdk.test.ts b/control-api/test/routes.internalPluginWorkloadSdk.test.ts
index 1ff7df43a..238514ac4 100644
--- a/control-api/test/routes.internalPluginWorkloadSdk.test.ts
+++ b/control-api/test/routes.internalPluginWorkloadSdk.test.ts
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import jwt from 'jsonwebtoken'
import request from 'supertest'
@@ -20,11 +20,16 @@ vi.mock('../src/services/pluginWorkloadSdkDb.js', () => ({
finalizePluginWorkloadSdkRevocation: (...args: unknown[]) => db.finalize(...args),
}))
-function sign(iss: 'wrc' | 'hcc'): string {
+function sign(
+ iss: 'wrc' | 'hcc',
+ jti = `${iss}-revocation-test`,
+ audience = 'control-api',
+ subject = `${iss}-provisioner`
+): string {
return jwt.sign(
- { iss, aud: 'control-api', sub: `${iss}-provisioner` },
+ { iss, aud: audience, sub: subject },
iss === 'wrc' ? config.internalControlJwtWrcHmacSecret : config.internalControlJwtHccHmacSecret,
- { algorithm: 'HS256', expiresIn: 60, jwtid: `${iss}-revocation-test` }
+ { algorithm: 'HS256', expiresIn: 60, jwtid: jti }
)
}
@@ -36,11 +41,21 @@ function app() {
}
describe('internal Plugin Workload SDK revocation', () => {
+ const originalLimit = config.pluginSdkInternalRlPerMin
beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))
+ config.pluginSdkInternalRlPerMin = 600
db.revoke.mockReset()
db.finalize.mockReset()
limiterQuery.mockReset()
- limiterQuery.mockResolvedValue({ rows: [{ count: 1 }], rowCount: 1 })
+ const counters = new Map<string, number>()
+ limiterQuery.mockImplementation(async (_sql: string, params: unknown[]) => {
+ const key = `${String(params[0])}|${String(params[1])}`
+ const count = (counters.get(key) ?? 0) + 1
+ counters.set(key, count)
+ return { rows: [{ count }], rowCount: 1 }
+ })
db.revoke.mockResolvedValue({
state: 'revoking',
revocationId: '11111111-1111-4111-8111-111111111111',
@@ -56,6 +71,11 @@ describe('internal Plugin Workload SDK revocation', () => {
})
})
+ afterEach(() => {
+ config.pluginSdkInternalRlPerMin = originalLimit
+ vi.useRealTimers()
+ })
+
it('revoke and finalize carry the authenticated WRC technical principal', async () => {
const authorization = `Bearer ${sign('wrc')}`
const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
@@ -106,4 +126,114 @@ describe('internal Plugin Workload SDK revocation', () => {
expect(response.status).toBe(403)
expect(db.revoke).not.toHaveBeenCalled()
})
+
+ it('keeps six hundred forbidden HCC requests on their signed principal without stealing the WRC allowance on the same IP', async () => {
+ const instance = app()
+ const hcc = `Bearer ${sign('hcc')}`
+ const wrc = `Bearer ${sign('wrc')}`
+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
+ for (let index = 0; index < 600; index += 1) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', hcc)
+ .send(binding)
+ .expect(403)
+ }
+ expect(db.revoke).not.toHaveBeenCalled()
+ for (let index = 0; index < 600; index += 1) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', wrc)
+ .send(binding)
+ .expect(200)
+ }
+ expect(db.revoke).toHaveBeenCalledTimes(600)
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', wrc)
+ .send(binding)
+ .expect(429)
+ }, 15_000)
+
+ it('keeps an exhausted invalid-audience or anonymous IP budget separate from the valid WRC allowance', async () => {
+ const instance = app()
+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
+ const invalidAudience = `Bearer ${sign('wrc', 'audience-negative', 'other-service')}`
+ for (let index = 0; index < 600; index += 1) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', invalidAudience)
+ .send(binding)
+ .expect(401)
+ }
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .send(binding)
+ .expect(429)
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', `Bearer ${sign('wrc')}`)
+ .send(binding)
+ .expect(200)
+ expect(db.revoke).toHaveBeenCalledTimes(1)
+ }, 15_000)
+
+ it('counts rotated signed credentials under one principal and recovers with PG at the minute boundary', async () => {
+ config.pluginSdkInternalRlPerMin = 2
+ const instance = app()
+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
+ for (let index = 0; index < 2; index += 1) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', `Bearer ${sign('wrc', `rotation-${index}`)}`)
+ .send(binding)
+ .expect(200)
+ }
+ const denied = await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', `Bearer ${sign('wrc', 'rotation-third')}`)
+ .send(binding)
+ expect(denied.status).toBe(429)
+ expect(denied.headers['retry-after']).toBe('43')
+ vi.setSystemTime(new Date('2026-10-02T12:01:00.000Z'))
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', `Bearer ${sign('wrc', 'rotation-next-minute')}`)
+ .send(binding)
+ .expect(200)
+ expect(db.revoke).toHaveBeenCalledTimes(3)
+ })
+
+ it('does not cap a higher verified internal override at the legacy IP ceiling of six hundred', async () => {
+ config.pluginSdkInternalRlPerMin = 601
+ const instance = app()
+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
+ const authorization = `Bearer ${sign('wrc')}`
+ for (let index = 0; index < 601; index += 1) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', authorization)
+ .send(binding)
+ .expect(200)
+ }
+ expect(db.revoke).toHaveBeenCalledTimes(601)
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', authorization)
+ .send(binding)
+ .expect(429)
+ }, 15_000)
+
+ it('keeps signed non-WRC subjects forbidden without turning an unusual subject into a server error', async () => {
+ const instance = app()
+ const binding = { recipeNamespace: 'sandbox-recipes', recipeName: 'sdk-recipe' }
+ for (const subject of ['service:ip:other', '\ud800']) {
+ await request(instance)
+ .post('/api/v1/internal/plugin-workload-sdk/revoke')
+ .set('Authorization', `Bearer ${sign('hcc', 'subject-negative', 'control-api', subject)}`)
+ .send(binding)
+ .expect(403)
+ }
+ expect(db.revoke).not.toHaveBeenCalled()
+ })
})
diff --git a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
index f50d28cf1..ac19373b0 100644
--- a/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
+++ b/control-api/test/routes.mcp-host.plugin-workload-sdk.test.ts
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import request from 'supertest'
+import { config } from '../src/config.js'
import { rateLimitPool } from '../src/db.js'
import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'
import * as codexConnection from '../src/services/codexSubscriptionConnection.js'
@@ -1453,7 +1454,10 @@ describe('POST /mcp-host/plugin-workload-sdk/credential-ticket/introspect', () =
// second query so this test remains focused on ticket work protection.
vi.mocked(rateLimitPool.query)
.mockResolvedValueOnce({ rows: [{ count: 1 }], rowCount: 1 } as never)
- .mockResolvedValueOnce({ rows: [{ count: 121 }], rowCount: 1 } as never)
+ .mockResolvedValueOnce({
+ rows: [{ count: config.pluginSdkCredentialRlPerMin + 1 }],
+ rowCount: 1,
+ } as never)
const res = await request(buildApp())
.post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect')
.set('Authorization', `Bearer ${issueSdkToken()}`)
diff --git a/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.ts b/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.ts
new file mode 100644
index 000000000..ec108f298
--- /dev/null
+++ b/control-api/test/routes.pluginWorkloadSdkCredentialBudget.test.ts
@@ -0,0 +1,220 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import { createMcpHostPluginWorkloadSdkRoutes } from '../src/routes/mcp-host/plugin-workload-sdk.routes.js'
+import { issuePluginWorkloadSdkCredentialTicket } from '../src/services/pluginWorkloadSdkCredentialTicket.js'
+import { hashPromptTargetPolicy } from '../src/services/pluginWorkloadSdkDb.js'
+import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'
+
+const boundary = vi.hoisted(() => ({
+ counts: new Map<string, number>(),
+ reissue: vi.fn(),
+ invocation: vi.fn(),
+ grant: vi.fn(),
+ receipt: vi.fn(),
+ attempt: vi.fn(),
+ redeem: vi.fn(),
+}))
+
+// The controlled limit exercises the real route guards at their configuration
+// boundary. Only database state and ticket-authorizer persistence are mocked;
+// JWT authentication, signed tickets, binding checks and quota middleware run.
+vi.mock('../src/config.js', async importOriginal => {
+ const actual = await importOriginal<typeof import('../src/config.js')>()
+ return { ...actual, config: { ...actual.config, pluginSdkCredentialRlPerMin: 6 } }
+})
+vi.mock('../src/db.js', () => ({
+ pool: { query: vi.fn(), connect: vi.fn() },
+ withTransaction: vi.fn(),
+ rateLimitPool: {
+ query: vi.fn(async (_sql: string, params: unknown[]) => {
+ const key = `${String(params[0])}|${String(params[1])}`
+ const count = (boundary.counts.get(key) ?? 0) + 1
+ boundary.counts.set(key, count)
+ return { rows: [{ count }], rowCount: 1 }
+ }),
+ },
+}))
+vi.mock('../src/services/pluginWorkloadSdkDb.js', async importOriginal => {
+ const actual = await importOriginal<typeof import('../src/services/pluginWorkloadSdkDb.js')>()
+ return {
+ ...actual,
+ getInvocationById: (...args: unknown[]) => boundary.invocation(...args),
+ findGrant: (...args: unknown[]) => boundary.grant(...args),
+ getPluginWorkloadSdkAttemptReceipt: (...args: unknown[]) => boundary.receipt(...args),
+ getPluginWorkloadSdkProviderAttempt: (...args: unknown[]) => boundary.attempt(...args),
+ redeemPluginWorkloadSdkCredentialTicketJti: (...args: unknown[]) => boundary.redeem(...args),
+ }
+})
+vi.mock('../src/services/pluginWorkloadSdkAuthorizer.js', async importOriginal => {
+ const actual =
+ await importOriginal<typeof import('../src/services/pluginWorkloadSdkAuthorizer.js')>()
+ return {
+ ...actual,
+ reissuePromptBridgeCredentialTicket: (...args: unknown[]) => boundary.reissue(...args),
+ }
+})
+
+const NS = 'sandbox-recipes'
+const RECIPE = 'credential-budget'
+const target = {
+ targetRef: 'primary-zai',
+ provider: 'zai',
+ model: 'glm-4.7',
+ credentialSlot: 'zai-api-key',
+}
+const policy = { policyRevision: 1, defaultTargetRef: target.targetRef, promptTargets: [target] }
+const policyHash = hashPromptTargetPolicy(policy)
+const attemptIds = [
+ '11111111-1111-4111-8111-111111111111',
+ '22222222-2222-4222-8222-222222222222',
+ '33333333-3333-4333-8333-333333333333',
+]
+
+describe('SDK credential-operation budget across actual endpoints', () => {
+ beforeEach(() => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(new Date('2026-10-02T12:00:17.500Z'))
+ boundary.counts.clear()
+ const used = new Set<string>()
+ boundary.redeem.mockReset().mockImplementation(async ({ jti }: { jti: string }) => {
+ if (used.has(jti)) return false
+ used.add(jti)
+ return true
+ })
+ boundary.grant.mockReset().mockResolvedValue({
+ id: 'grant-credential-budget',
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ capabilityFamily: 'promptBridge',
+ ...policy,
+ policyState: 'active',
+ })
+ boundary.invocation.mockReset().mockImplementation(async (id: string) => ({
+ id,
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ method: 'promptBridge',
+ status: 'in_progress',
+ attemptGeneration: 1,
+ authorizationDecision: 'authorized',
+ promptAuthorization: {
+ policyRevision: 1,
+ policyHash,
+ authorizedTargetRefs: [target.targetRef],
+ },
+ }))
+ boundary.receipt.mockReset().mockImplementation(async (invocationId: string) => ({
+ invocationId,
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ method: 'promptBridge',
+ status: 'in_progress',
+ attemptGeneration: 1,
+ }))
+ boundary.attempt.mockReset().mockImplementation(async (id: string) => ({
+ id,
+ invocationId: `inv-${attemptIds.indexOf(id) + 1}`,
+ attemptGeneration: 1,
+ targetRef: target.targetRef,
+ status: 'in_progress',
+ }))
+ boundary.reissue
+ .mockReset()
+ .mockImplementation(async ({ invocationId }: { invocationId: string }) => {
+ const index = Number(invocationId.slice(4)) - 1
+ return {
+ ok: true,
+ value: {
+ invocationId,
+ attemptGeneration: 1,
+ targetRef: target.targetRef,
+ providerAttemptId: attemptIds[index],
+ providerAttemptIndex: 1,
+ credentialTicket: issuePluginWorkloadSdkCredentialTicket({
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptId: attemptIds[index],
+ providerAttemptIndex: 1,
+ target,
+ policyRevision: 1,
+ policyHash,
+ }),
+ policyRevision: 1,
+ policyHash,
+ expiresInSeconds: 60,
+ },
+ }
+ })
+ })
+
+ afterEach(() => vi.useRealTimers())
+
+ it('permits two three-operation static credential sequences at six, then rejects the third reissue before ticket work', async () => {
+ const instance = express()
+ instance.use(express.json())
+ instance.use(createMcpHostPluginWorkloadSdkRoutes())
+ const authorization = `Bearer ${
+ issueMcpHostAccessJwt(NS, RECIPE, undefined, {
+ workflowControlScopes: ['plugin-workload-sdk'],
+ }).token
+ }`
+ for (let index = 1; index <= 2; index += 1) {
+ const invocationId = `inv-${index}`
+ const issued = await request(instance)
+ .post('/mcp-host/plugin-workload-sdk/prompt-bridge/credential-ticket')
+ .set('Authorization', authorization)
+ .send({
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ invocationId,
+ targetRef: target.targetRef,
+ attemptGeneration: 1,
+ })
+ expect(issued.status).toBe(201)
+ const body = {
+ credentialTicket: issued.body.credentialTicket,
+ invocationId,
+ targetRef: target.targetRef,
+ attemptGeneration: 1,
+ providerAttemptId: attemptIds[index - 1],
+ providerAttemptIndex: 1,
+ }
+ await request(instance)
+ .post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect')
+ .set('Authorization', authorization)
+ .send({ ...body, redeem: false })
+ .expect(200)
+ expect(boundary.redeem).toHaveBeenCalledTimes(index - 1)
+ await request(instance)
+ .post('/mcp-host/plugin-workload-sdk/credential-ticket/introspect')
+ .set('Authorization', authorization)
+ .send({ ...body, redeem: true })
+ .expect(200)
+ expect(boundary.redeem).toHaveBeenCalledTimes(index)
+ }
+ const denied = await request(instance)
+ .post('/mcp-host/plugin-workload-sdk/prompt-bridge/credential-ticket')
+ .set('Authorization', authorization)
+ .send({
+ recipeNamespace: NS,
+ recipeName: RECIPE,
+ invocationId: 'inv-3',
+ targetRef: target.targetRef,
+ attemptGeneration: 1,
+ })
+ expect(denied.status).toBe(429)
+ expect(Number(denied.headers['retry-after'])).toBeGreaterThan(0)
+ expect(boundary.reissue).toHaveBeenCalledTimes(2)
+ expect(boundary.invocation).toHaveBeenCalledTimes(4)
+ expect(boundary.redeem).toHaveBeenCalledTimes(2)
+ const credentialCounts = [...boundary.counts.entries()].filter(([key]) =>
+ key.startsWith('plugin_workload_sdk_credential:')
+ )
+ expect(credentialCounts).toHaveLength(1)
+ expect(credentialCounts[0][0]).toContain(`plugin_workload_sdk_credential:${NS}/${RECIPE}|`)
+ expect(credentialCounts[0][1]).toBe(7)
+ })
+})
diff --git a/control-api/test/routes.registryConnect.test.ts b/control-api/test/routes.registryConnect.test.ts
index 07b681dad..e540db6d5 100644
--- a/control-api/test/routes.registryConnect.test.ts
+++ b/control-api/test/routes.registryConnect.test.ts
@@ -8,6 +8,9 @@ import { checkAndIncrement } from '../src/services/rateLimiterService.js'
const { cfg } = vi.hoisted(() => ({
cfg: {
+ adminRegistryConnectStatusPerMin: 150,
+ adminRegistryConnectRequestPerMin: 15,
+ adminRegistryConnectRecoveryPerMin: 50,
registryConnectionMode: 'self-hosted',
registryUrl: 'https://example.com',
registryAuthEnabled: true,
@@ -513,8 +516,8 @@ describe('registry connect flow', () => {
connDb.getRegistryConnection.mockResolvedValue(null)
const res = await request(app()).get('/admin/registry/connect').expect(200)
expect(res.body.state).toBe('disconnected')
- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_status:admin-uuid-123', 30)
- expect(res.headers['x-ratelimit-limit']).toBe('30')
+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_status:admin-uuid-123', 150)
+ expect(res.headers['x-ratelimit-limit']).toBe('150')
})
// Proves the limiter blocks BEFORE any DB/registry work: a denied admin
@@ -549,8 +552,8 @@ describe('registry connect flow', () => {
.send({ requested_org_name: 'acme', contact_email: '[email protected]' })
.expect(202)
expect(res.body.state).toBe('pending')
- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_request:admin-uuid-123', 3)
- expect(res.headers['x-ratelimit-limit']).toBe('3')
+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_request:admin-uuid-123', 15)
+ expect(res.headers['x-ratelimit-limit']).toBe('15')
})
// Proves the limiter blocks BEFORE any registry work: a denied admin must
@@ -1018,8 +1021,8 @@ describe('POST recover', () => {
})
const res = await request(app()).post('/admin/registry/connect/recover').expect(200)
expect(res.body).toMatchObject({ state: 'connected', org: 'acme' })
- expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_recover:admin-uuid-123', 10)
- expect(res.headers['x-ratelimit-limit']).toBe('10')
+ expect(checkAndIncrement).toHaveBeenCalledWith('registry_connect_recover:admin-uuid-123', 50)
+ expect(res.headers['x-ratelimit-limit']).toBe('50')
})
// Proves the limiter actually blocks, and blocks BEFORE any registry work:
diff --git a/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.ts b/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.ts
new file mode 100644
index 000000000..e96dd8a6e
--- /dev/null
+++ b/control-api/test/services.adminSubscriptionRateLimitMigration.realPostgres.integration.test.ts
@@ -0,0 +1,232 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'
+import { createHash, randomBytes } from 'node:crypto'
+import { Pool, type PoolClient } from 'pg'
+import { applyAdminSubscriptionRateLimitNamespace } from '../src/services/adminSubscriptionRateLimitMigration.js'
+import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+describeRealPostgres('administrative subscription counter namespace migration', () => {
+ const database = `admin_quota_migration_${randomBytes(6).toString('hex')}`
+ let adminPool: Pool
+ let pool: Pool
+ let client: PoolClient
+ const windowStart = 1_800_000_000_000
+ const prefixes = [
+ ['admin_codex_read:', 'admin_subscription_read:'],
+ ['admin_codex_write:', 'admin_subscription_write:'],
+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],
+ ] as const
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE "${database}"`)
+ const url = new URL(adminUrl!)
+ url.pathname = `/${database}`
+ pool = new Pool({ connectionString: url.toString() })
+ client = await pool.connect()
+ }, 60_000)
+
+ beforeEach(async () => {
+ await client.query('DROP TABLE IF EXISTS rate_limit_buckets CASCADE')
+ await client.query(`CREATE TABLE rate_limit_buckets (
+ bucket_key TEXT NOT NULL,
+ window_start_ms BIGINT NOT NULL,
+ count INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (bucket_key, window_start_ms)
+ )`)
+ })
+
+ afterAll(async () => {
+ client?.release()
+ await pool?.end()
+ if (!adminPool) return
+ await adminPool.query(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`)
+ await adminPool.end()
+ })
+
+ async function migrate() {
+ await client.query('BEGIN')
+ try {
+ await applyAdminSubscriptionRateLimitNamespace(client)
+ await client.query('COMMIT')
+ } catch (error) {
+ await client.query('ROLLBACK')
+ throw error
+ }
+ }
+
+ it.each(prefixes)(
+ 'merges existing %s accounting without a new allowance',
+ async (legacy, current) => {
+ // Synthetic suffixes model ledger identities, not authentication tokens.
+ await client.query(
+ `INSERT INTO rate_limit_buckets VALUES
+ ($1, $3, 31), ($2, $3, 9), ('unrelated:fixture', $3, 7)`,
+ [`${legacy}state:fixture`, `${current}state:fixture`, windowStart]
+ )
+ await migrate()
+ const rows = await client.query(
+ 'SELECT bucket_key, count FROM rate_limit_buckets ORDER BY bucket_key'
+ )
+ expect(rows.rows).toEqual([
+ { bucket_key: `${current}state:fixture`, count: 40 },
+ { bucket_key: 'unrelated:fixture', count: 7 },
+ ])
+ await migrate()
+ const repeated = await client.query(
+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1',
+ [`${current}state:fixture`]
+ )
+ expect(repeated.rows[0].count).toBe(40)
+ }
+ )
+
+ it.each(prefixes)('normalizes writes from an older %s binary', async (legacy, current) => {
+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 31)', [
+ `${legacy}fixture`,
+ windowStart,
+ ])
+ await migrate()
+ const check = await checkAndIncrementWithQuery(
+ (sql, values) => client.query(sql, values),
+ `${legacy}fixture`,
+ 150,
+ windowStart
+ )
+ expect(check.count).toBe(32)
+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')
+ expect(rows.rows).toEqual([{ bucket_key: `${current}fixture`, count: 32 }])
+ })
+
+ it('keeps concurrent old and new writers on one physical counter', async () => {
+ await migrate()
+ const results = await Promise.all(
+ Array.from({ length: 40 }, (_, index) =>
+ checkAndIncrementWithQuery(
+ (sql, values) => pool.query(sql, values),
+ `${index % 2 ? 'admin_codex_read:' : 'admin_subscription_read:'}fixture`,
+ 32,
+ windowStart
+ )
+ )
+ )
+ expect(results.filter(result => result.allowed)).toHaveLength(32)
+ expect(new Set(results.map(result => result.count)).size).toBe(40)
+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')
+ expect(rows.rows).toEqual([{ bucket_key: 'admin_subscription_read:fixture', count: 40 }])
+ })
+
+ it.each(prefixes)(
+ 'retains opaque long-key accounting for %s',
+ async (legacyPrefix, currentPrefix) => {
+ const suffix = `state:${'x'.repeat(600)}`
+ const legacy = legacyPrefix + suffix
+ const current = currentPrefix + suffix
+ const digest = `sha256-long-key:${createHash('sha256').update(legacy).digest('hex')}`
+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $3, 31), ($2, $3, 9)', [
+ digest,
+ current,
+ windowStart,
+ ])
+ await migrate()
+ const renamed = await checkAndIncrementWithQuery(
+ (sql, values) => client.query(sql, values),
+ current,
+ 150,
+ windowStart
+ )
+ expect(renamed.count).toBe(41)
+ const older = await checkAndIncrementWithQuery(
+ (sql, values) => client.query(sql, values),
+ digest,
+ 150,
+ windowStart
+ )
+ expect(older.count).toBe(42)
+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')
+ expect(rows.rows).toEqual([{ bucket_key: digest, count: 42 }])
+ }
+ )
+
+ it('recovers at the next actual SQL window without deleting consumed counts', async () => {
+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 150)', [
+ 'admin_codex_read:fixture',
+ windowStart,
+ ])
+ await migrate()
+ const run = (now: number) =>
+ checkAndIncrementWithQuery(
+ (sql, values) => client.query(sql, values),
+ 'admin_subscription_read:fixture',
+ 150,
+ now
+ )
+ expect((await run(windowStart)).allowed).toBe(false)
+ const next = await run(windowStart + 60_000)
+ expect(next.allowed).toBe(true)
+ expect(next.count).toBe(1)
+ const previous = await client.query(
+ 'SELECT count FROM rate_limit_buckets WHERE window_start_ms = $1',
+ [windowStart]
+ )
+ expect(previous.rows[0].count).toBe(151)
+ })
+
+ it('retains exact suffixes and independently counted windows', async () => {
+ const suffix = 'state:fixture_%:admin_codex_read:embedded'
+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $2, 4), ($1, $3, 6)', [
+ `admin_codex_read:${suffix}`,
+ windowStart,
+ windowStart + 60_000,
+ ])
+ await migrate()
+ const rows = await client.query(
+ 'SELECT bucket_key, window_start_ms, count FROM rate_limit_buckets ORDER BY window_start_ms'
+ )
+ expect(rows.rows).toEqual([
+ {
+ bucket_key: `admin_subscription_read:${suffix}`,
+ window_start_ms: String(windowStart),
+ count: 4,
+ },
+ {
+ bucket_key: `admin_subscription_read:${suffix}`,
+ window_start_ms: String(windowStart + 60_000),
+ count: 6,
+ },
+ ])
+ })
+
+ it('preserves accounting when the longer namespace crosses the 512-byte boundary', async () => {
+ const suffix = 'x'.repeat(495)
+ const legacy = 'admin_codex_read:' + suffix
+ const current = 'admin_subscription_read:' + suffix
+ expect(Buffer.byteLength(legacy)).toBe(512)
+ expect(Buffer.byteLength(current)).toBeGreaterThan(512)
+ await client.query('INSERT INTO rate_limit_buckets VALUES ($1, $3, 31), ($2, $3, 9)', [
+ legacy,
+ current,
+ windowStart,
+ ])
+ await migrate()
+ const expected = `sha256-long-key:${createHash('sha256').update(legacy).digest('hex')}`
+ const result = await checkAndIncrementWithQuery(
+ (sql, values) => client.query(sql, values),
+ current,
+ 150,
+ windowStart
+ )
+ expect(result.count).toBe(41)
+ // An older binary's short raw input is canonicalized by the invoker trigger.
+ await client.query(
+ `INSERT INTO rate_limit_buckets VALUES ($1, $2, 1)
+ ON CONFLICT (bucket_key, window_start_ms) DO UPDATE
+ SET count = rate_limit_buckets.count + EXCLUDED.count`,
+ [legacy, windowStart]
+ )
+ const rows = await client.query('SELECT bucket_key, count FROM rate_limit_buckets')
+ expect(rows.rows).toEqual([{ bucket_key: expected, count: 42 }])
+ })
+})
diff --git a/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts b/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts
index f10ac6885..fd3feb5d4 100644
--- a/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkQuotaTracker.test.ts
@@ -52,7 +52,7 @@ const grant = (
// ─── Issue #348 (plan D3) — platform per-minute defaults ─────────────────
//
// RED-FIRST (plan §6.6): these tests assert the POST-change ceilings
-// (promptBridge 120/min, clientNotifications 150/min, sourced from
+// (promptBridge 600/min, clientNotifications 750/min, sourced from
// config.pluginSdkPromptBridgeRlPerMin / config.pluginSdkNotificationsRlPerMin).
// Each case is POSITIVE (at the ceiling → allowed) AND NEGATIVE (one over → denied
// with quota_exceeded). Against pre-change code the hardcoded defaults are 60/120,
@@ -71,32 +71,32 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =
vi.mocked(sdkDb.countRecentInvocations).mockReset()
})
- it('allows promptBridge at exactly 120/minute (positive) and denies at 121 (negative, platform default)', async () => {
+ it('allows promptBridge at exactly 600/minute (positive) and denies at 601 (negative, platform default)', async () => {
const noOverrides = grant({ quotaLimits: {} })
// POSITIVE: at the ceiling → allowed.
- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(120)
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(600)
await expect(
checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)
).resolves.toEqual({ ok: true })
// NEGATIVE: one over the ceiling → denied.
- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(121)
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(601)
await expect(
checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)
).resolves.toMatchObject({
ok: false,
error: 'quota_exceeded',
retryable: false,
- message: expect.stringContaining('120/minute'),
+ message: expect.stringContaining('600/minute'),
})
})
- it('allows clientNotifications at exactly 150/minute (positive) and denies at 151 (negative), narrowed to the eventType', async () => {
+ it('allows clientNotifications at exactly 750/minute (positive) and denies at 751 (negative), narrowed to the eventType', async () => {
const noOverrides = grant({ capabilityFamily: 'clientNotifications', quotaLimits: {} })
// POSITIVE: at the ceiling → allowed.
- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(150)
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(750)
await expect(
checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', noOverrides, {
eventType: 'lead.followup.due',
@@ -110,7 +110,7 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =
)
// NEGATIVE: one over the ceiling → denied.
- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(151)
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(751)
await expect(
checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', noOverrides, {
eventType: 'lead.followup.due',
@@ -119,14 +119,14 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =
ok: false,
error: 'quota_exceeded',
retryable: false,
- message: expect.stringContaining('150/minute'),
+ message: expect.stringContaining('750/minute'),
})
})
- it('frees the recipe once the trailing window drains (deny at 121, allow at 0)', async () => {
+ it('frees the recipe once the trailing window drains (deny at 601, allow at 0)', async () => {
const noOverrides = grant({ quotaLimits: {} })
- vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(121)
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(601)
await expect(
checkRateLimit('sandbox-recipes', 'sdk-recipe', 'promptBridge', noOverrides)
).resolves.toMatchObject({ ok: false, error: 'quota_exceeded' })
@@ -158,4 +158,29 @@ describe('checkRateLimit — platform per-minute rate limits (issue #348)', () =
message: expect.stringContaining('5/minute'),
})
})
+
+ it('lets a grant-level maxNotificationsPerMinute override win over the platform default', async () => {
+ const withOverride = grant({
+ capabilityFamily: 'clientNotifications',
+ quotaLimits: { maxNotificationsPerMinute: 9 },
+ })
+
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(9)
+ await expect(
+ checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', withOverride, {
+ eventType: 'lead.followup.due',
+ })
+ ).resolves.toEqual({ ok: true })
+
+ vi.mocked(sdkDb.countRecentInvocations).mockResolvedValue(10)
+ await expect(
+ checkRateLimit('sandbox-recipes', 'sdk-recipe', 'clientNotifications', withOverride, {
+ eventType: 'lead.followup.due',
+ })
+ ).resolves.toMatchObject({
+ ok: false,
+ error: 'quota_exceeded',
+ message: expect.stringContaining('9/minute'),
+ })
+ })
})
diff --git a/control-api/test/services.rateLimitBucketKey.test.ts b/control-api/test/services.rateLimitBucketKey.test.ts
new file mode 100644
index 000000000..5f99eaab3
--- /dev/null
+++ b/control-api/test/services.rateLimitBucketKey.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest'
+import { createHash } from 'node:crypto'
+import {
+ MAX_RATE_LIMIT_BUCKET_KEY_BYTES,
+ boundedBucketKey,
+} from '../src/services/rateLimitBucketKey.js'
+
+describe('rate-limit bucket storage identity', () => {
+ const aliases = [
+ ['admin_codex_read:', 'admin_subscription_read:'],
+ ['admin_codex_write:', 'admin_subscription_write:'],
+ ['codex_oauth_callback:', 'subscription_oauth_callback:'],
+ ] as const
+
+ it.each(aliases)(
+ 'preserves the opaque legacy %s identity for long renamed keys',
+ (oldPrefix, newPrefix) => {
+ const suffix = 'fixture_'.repeat(100)
+ const expected = `sha256-long-key:${createHash('sha256')
+ .update(oldPrefix + suffix)
+ .digest('hex')}`
+ expect(boundedBucketKey(oldPrefix + suffix)).toBe(expected)
+ expect(boundedBucketKey(newPrefix + suffix)).toBe(expected)
+ }
+ )
+
+ it('keeps unrelated and embedded namespace text unchanged', () => {
+ const key = 'unrelated:admin_subscription_read:' + 'fixture_'.repeat(100)
+ expect(boundedBucketKey(key)).toBe(
+ `sha256-long-key:${createHash('sha256').update(key).digest('hex')}`
+ )
+ })
+
+ it('keeps already bounded neutral namespaces for the database migration', () => {
+ const key = 'admin_subscription_read:fixture'
+ expect(boundedBucketKey(key)).toBe(key)
+ })
+
+ it('uses the byte boundary and keeps stored keys bounded', () => {
+ const oldKey = 'admin_codex_read:' + 'é'.repeat(MAX_RATE_LIMIT_BUCKET_KEY_BYTES)
+ const newKey = 'admin_subscription_read:' + 'é'.repeat(MAX_RATE_LIMIT_BUCKET_KEY_BYTES)
+ expect(boundedBucketKey(newKey)).toBe(boundedBucketKey(oldKey))
+ expect(Buffer.byteLength(boundedBucketKey(newKey))).toBeLessThan(
+ MAX_RATE_LIMIT_BUCKET_KEY_BYTES
+ )
+ })
+})
diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.ts
index 1d6960666..a6f2028aa 100644
--- a/control-api/test/workflows.shared.rateLimit.test.ts
+++ b/control-api/test/workflows.shared.rateLimit.test.ts
@@ -1,19 +1,20 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
+import { createHash } from 'node:crypto'
import request from 'supertest'
import { config } from '../src/config.js'
import {
- adminCodexReadRateLimits,
- adminCodexWriteRateLimits,
adminOutputsReadRateLimits,
+ adminSubscriptionReadRateLimits,
+ adminSubscriptionWriteRateLimits,
adminWorkflowRateLimitCredential,
adminWorkflowTriggerRateLimit,
- codexOAuthCallbackRateLimits,
llmProviderAttemptAuthorizeRateLimits,
mcpHostAttemptRateLimitKey,
mcpHostWorkflowTriggerRateLimit,
mcpHostWorkflowTriggerRateLimitCredential,
shouldSkipWorkflowGrantEdgeRateLimit,
+ subscriptionOAuthCallbackRateLimits,
verifiedAdminRateLimitSubject,
workflowAdminReadRateLimits,
workflowGrantEdgeRateLimitKey,
@@ -99,7 +100,7 @@ describe('routes/workflows/shared/rateLimit', () => {
const counts = countRequests()
const app = express()
for (const path of ['/codex/connections', '/grok/connections', '/grok/models']) {
- app.get(path, ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ app.get(path, ...adminSubscriptionReadRateLimits(), (_req, res) => res.sendStatus(204))
}
const paths = ['/codex/connections', '/grok/connections', '/grok/models']
for (let i = 0; i < 150; i++) {
@@ -126,7 +127,9 @@ describe('routes/workflows/shared/rateLimit', () => {
it('gives distinct verified sessions independent subscription allowances', async () => {
countRequests()
const app = express()
- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
for (const session of ['signed-admin-a', 'signed-admin-b']) {
for (let i = 0; i < 150; i++) {
await request(app)
@@ -141,13 +144,37 @@ describe('routes/workflows/shared/rateLimit', () => {
.expect(429)
})
+ it('aligns edge recovery with the database wall-clock window after a late first request', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(1_800_000_030_000)
+ mockVerifyAdminToken.mockImplementation(() => signedClaims('window-fixture'))
+ countRequests()
+ const app = express()
+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
+ const cookie = 'control_ui_admin_session=window-fixture'
+ const first = await request(app).get('/connections').set('Cookie', cookie).expect(204)
+ expect(first.headers.ratelimit).toMatch(/reset=30\b/)
+ for (let i = 1; i < 150; i++)
+ await request(app).get('/connections').set('Cookie', cookie).expect(204)
+ const denied = await request(app).get('/connections').set('Cookie', cookie).expect(429)
+ expect(denied.headers['retry-after']).toBe('30')
+ vi.setSystemTime(1_800_000_060_000)
+ for (let i = 0; i < 150; i++)
+ await request(app).get('/connections').set('Cookie', cookie).expect(204)
+ await request(app).get('/connections').set('Cookie', cookie).expect(429)
+ })
+
it('uses a configured read budget for both edge and distributed gates', async () => {
const original = config.adminSubscriptionReadPerMin
config.adminSubscriptionReadPerMin = 3
try {
countRequests()
const app = express()
- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
for (let i = 0; i < 3; i++) {
await request(app)
.get('/connections')
@@ -165,11 +192,36 @@ describe('routes/workflows/shared/rateLimit', () => {
}
})
+ it.each(['signed-admin-b', 'unverified-fixture'])(
+ 'attributes subscription reads to the cookie despite an extra bearer %s',
+ async extra => {
+ countRequests()
+ const app = express()
+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
+ await request(app)
+ .get('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .set('Authorization', `Bearer ${extra}`)
+ .expect(204)
+ const expected = createHash('sha256').update('signed-admin-a').digest('hex').slice(0, 32)
+ expect(mockCheckAndIncrement.mock.calls[0]?.slice(0, 2)).toEqual([
+ `admin_subscription_read:${expected}`,
+ 150,
+ ])
+ }
+ )
+
it('allows 100 subscription writes without consuming the read quota', async () => {
const counts = countRequests()
const app = express()
- app.post('/connections', ...adminCodexWriteRateLimits(), (_req, res) => res.sendStatus(204))
- app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ app.post('/connections', ...adminSubscriptionWriteRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
+ app.get('/connections', ...adminSubscriptionReadRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
for (let i = 0; i < 100; i++) {
await request(app)
.post('/connections')
@@ -190,7 +242,9 @@ describe('routes/workflows/shared/rateLimit', () => {
it('enforces the 100 callback budget with retry timing', async () => {
countRequests()
const app = express()
- app.get('/callback', ...codexOAuthCallbackRateLimits(), (_req, res) => res.sendStatus(204))
+ app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
for (let i = 0; i < 100; i++) {
await request(app).get('/callback').query({ state: 'unit-callback-state' }).expect(204)
}
@@ -202,6 +256,40 @@ describe('routes/workflows/shared/rateLimit', () => {
expect(denied.headers['retry-after']).toBeDefined()
})
+ it.each([
+ ['workflow grant reads', 300, workflowGrantReadRateLimits],
+ ['workflow grant writes', 100, workflowGrantWriteRateLimits],
+ ['workflow administrative reads', 300, workflowAdminReadRateLimits],
+ ['administrative output reads', 150, adminOutputsReadRateLimits],
+ ] as const)(
+ 'allows the increased %s capacity through both gates',
+ async (_family, limit, factory) => {
+ mockVerifyAdminToken.mockImplementation(() => signedClaims('capacity-fixture'))
+ countRequests()
+ const app = express()
+ app.get('/capacity', ...factory(), (_req, res) => res.sendStatus(204))
+ const cookie = 'control_ui_admin_session=capacity-fixture'
+ for (let i = 0; i < limit; i++) {
+ await request(app).get('/capacity').set('Cookie', cookie).expect(204)
+ }
+ const denied = await request(app).get('/capacity').set('Cookie', cookie).expect(429)
+ expect(denied.body.code).toBe('rate_limited')
+ expect(denied.headers['retry-after']).toBeDefined()
+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === limit)).toBe(true)
+ }
+ )
+
+ it('retains the 20-request callback IP safeguard when no state is supplied', async () => {
+ countRequests()
+ const app = express()
+ app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>
+ res.sendStatus(204)
+ )
+ for (let i = 0; i < 20; i++) await request(app).get('/callback').expect(204)
+ await request(app).get('/callback').expect(429)
+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === 20)).toBe(true)
+ })
+
beforeEach(() => {
mockVerifyAdminToken.mockImplementation((token: string) =>
token.startsWith('signed-') ? signedClaims(token.slice('signed-'.length)) : null
@@ -363,9 +451,9 @@ describe('routes/workflows/shared/rateLimit', () => {
expect(workflowGrantWriteRateLimits()).toHaveLength(2)
expect(workflowAdminReadRateLimits()).toHaveLength(2)
expect(adminOutputsReadRateLimits()).toHaveLength(2)
- expect(adminCodexReadRateLimits()).toHaveLength(2)
- expect(adminCodexWriteRateLimits()).toHaveLength(2)
- expect(codexOAuthCallbackRateLimits()).toHaveLength(2)
+ expect(adminSubscriptionReadRateLimits()).toHaveLength(2)
+ expect(adminSubscriptionWriteRateLimits()).toHaveLength(2)
+ expect(subscriptionOAuthCallbackRateLimits()).toHaveLength(2)
expect(llmProviderAttemptAuthorizeRateLimits()).toHaveLength(2)
})
diff --git a/deploy/base/control-plane/configmaps.yaml b/deploy/base/control-plane/configmaps.yaml
index 4ef42b1ec..75434f139 100644
--- a/deploy/base/control-plane/configmaps.yaml
+++ b/deploy/base/control-plane/configmaps.yaml
@@ -60,18 +60,44 @@ data:
WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_ENABLED: "true"
WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_INTERVAL_MS: "5000"
WORKFLOW_APPROVAL_NOTIFICATION_DELIVERY_BATCH_SIZE: "10"
- APPROVAL_RL_REQUEST_PER_MIN: "120"
+ APPROVAL_RL_REQUEST_PER_MIN: "600"
+ APPROVAL_RL_REFRESH_PER_MIN: "100"
+ APPROVAL_RL_REISSUE_PER_MIN: "25"
+ CONTROL_API_OAUTH_BROKER_RL_PER_MIN: "300"
+ CONTROL_API_HOST_WAKE_RL_PER_MIN: "150"
# Plugin Workload SDK platform rate limits (issue #348). Per-run SDK quotas
# (maxRequestsPerRun / maxNotificationsPerRun) are deprecated and ignored;
- # limits are enforced over a trailing 60s window (no fixed reset); once
- # traffic slows below the limit the window drains within ~60s — stepless
- # recipes are never permanently trapped. Values MUST equal the code defaults
- # in control-api/src/config.ts (150/120/600/600) — the config deploy-mirror
- # test fails on drift.
- CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "150"
- CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "120"
- CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "600"
+ # Method quotas use their existing trailing 60s window. Authenticated request
+ # and edge budgets use the PG calendar minute; anonymous IP backstops retain
+ # first-hit windows. Defaults below mirror control-api/src/config.ts.
+ # Healthy static prompts cost three credential operations and six/seven SDK
+ # API requests; the 1800/6000 defaults keep the prompt600 allowance reachable
+ # without removing fresh capability proof, either Secret check or redemption.
+ CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "750"
+ CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "6000"
CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: "6000"
+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: "1800"
+ CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN: "300"
+ CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN: "300"
+ CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN: "100"
+ CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN: "50"
+ CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN: "150"
+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN: "300"
+ CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_ANONYMOUS_IP_PER_MIN: "60"
+ CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN: "150"
+ CONTROL_API_ADMIN_REGISTRY_GRANTS_PER_MIN: "150"
+ CONTROL_API_ADMIN_REGISTRY_CONNECT_STATUS_PER_MIN: "150"
+ CONTROL_API_ADMIN_REGISTRY_CONNECT_REQUEST_PER_MIN: "15"
+ CONTROL_API_ADMIN_REGISTRY_CONNECT_RECOVERY_PER_MIN: "50"
+ CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN: "300"
+ CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN: "150"
+ CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN: "150"
+ CONTROL_API_ADMIN_GFS_SHARES_PER_MIN: "150"
+ CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN: "150"
# Hard ceiling for caller-provided ttlSeconds. Defaults to 48h so a client
# asking for ttlSeconds=604800 (code default) can't freeze a workflow step
# for a full week. Overlays may tighten further for stricter environments.
diff --git a/deploy/overlays/minikube/configmaps/control-api-config.yaml b/deploy/overlays/minikube/configmaps/control-api-config.yaml
index 91c9fa842..21c7ba16b 100644
--- a/deploy/overlays/minikube/configmaps/control-api-config.yaml
+++ b/deploy/overlays/minikube/configmaps/control-api-config.yaml
@@ -89,10 +89,14 @@ data:
# trailing 60s window (no fixed reset); once traffic slows below the limit
# the window drains within ~60s — stepless recipes are never permanently
# trapped.
- CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "150"
- CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "120"
- CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN: "750"
+ CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN: "6000"
CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN: "6000"
+ CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN: "600"
+ CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN: "1800"
# OSS default: point at the shared registry so a self-hoster browses the
# public catalog out of the box and can run the connect flow (register →
# approve → claim) against it. self-hosted is the only mode that fits an OSS
diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.md
new file mode 100644
index 000000000..70c0ff2fa
--- /dev/null
+++ b/docs/architecture/administrative-rate-limits.md
@@ -0,0 +1,147 @@
+# Administrative rate limits and coordinated access
+
+Control API owns request quotas. Control UI reduces redundant optional reads
+and explains genuine throttling. A person’s action can generate several HTTP
+requests, and each request can traverse several independent counters.
+
+For one workload sharing the relevant principal and window, the action bound
+is the minimum of each applicable budget divided by the operations that
+action charges to it. A requests-per-minute setting is not a throughput
+guarantee. Retries, failover and other methods consume additional allowance.
+
+## Administrative browser path
+
+Browser -> public ingress -> Control UI Next proxy -> Control API admin
+authentication -> local edge guard -> shared PostgreSQL counter -> response.
+
+The checked-in UI proxy has no requests-per-minute quota and forwards status,
+body, Retry-After and quota headers. Checked-in Control UI calls Control API
+directly; the repository’s nginx gateway configs contain no limit_req or
+limit_conn on this path. Live Cloudflare/WAF policy, forwarding-IP trust and
+actual deployed ENV owners remain deployment checks.
+
+Subscription administration and capability discovery use the cookie accepted
+by the parent administrator middleware. Quota attribution follows that same
+cookie even when an unrelated Authorization header is present. Workflow
+administration preserves its separate existing Bearer/cookie transport.
+
+Read quotas are shared across subscription providers and capability discovery.
+Writes have a separate quota. Neither family creates a global aggregate of
+every administrator. Existing signed-session hashing and verified Host/recipe
+bindings are preserved; a naming change does not choose a new identity policy.
+
+## Configuration defaults
+
+All values below count operations per minute unless stated otherwise. Valid
+explicit numeric ENV and grant overrides remain authoritative; deliberately
+lower overrides lower the effective workload capacity.
+
+| Family | Configuration/default | Binding and inner coordination |
+| --- | --- | --- |
+| Subscription reads | CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN=150 | Same signed browser credential across providers/capabilities; aligned edge and ledger. |
+| Subscription writes | CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN=100 | Separate from reads; no automatic replay. |
+| Subscription callback | CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN=100 | Per supplied state; no-state IP safeguard20. Presence is not state validation. |
+| Workflow administrative reads | CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN=300 | Existing verified administrator credential. |
+| Workflow grant reads/writes | CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN=300; CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN=100 | Separate families and aligned verified edge/ledger. |
+| Administrative workflow trigger | CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN=50 | Existing verified caller. External/MCP trigger contract remains10. |
+| Outputs reads | CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN=150 | Existing administrator binding. |
+| Host wake | CONTROL_API_HOST_WAKE_RL_PER_MIN=150 | Verified Host reference after authentication and Host match. |
+| Registry keys/grants/status/connect/recovery | CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN=150; GRANTS=150; CONNECT_STATUS=150; CONNECT_REQUEST=15; CONNECT_RECOVERY=50 | Separate authenticated administrator-subject families; each suffix uses the CONTROL_API_ADMIN_REGISTRY_ prefix. |
+| Connector deletion | CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN=300; CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN=150 | PostgreSQL150 controls the authenticated allowance. Authorization is preserved. |
+| SDK admin/internal | CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN=600; CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN=600 | Verified principal; internal signed iss/sub separated from invalid/anonymous IP600. |
+| SDK authenticated request/preauth | CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN=6000; CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN=6000 | Verified runtime Host/recipe; aligned calendar minute. |
+| SDK anonymous preauth | CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN=600 | Source-IP first-hit backstop; a valid attributed runtime JWT does not consume it. |
+| SDK credential operations | CONTROL_API_PLUGIN_SDK_CREDENTIAL_PER_MIN=1800 | Reissue and both introspections share this quota. Ticket authority/binding and single use remain. |
+| SDK prompt/notification defaults | CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN=600; CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN=750 | Explicit grant overrides remain; existing method-quota window semantics remain. |
+| Approval request/refresh/reissue | APPROVAL_RL_REQUEST_PER_MIN=600; APPROVAL_RL_REFRESH_PER_MIN=100; APPROVAL_RL_REISSUE_PER_MIN=25 | Existing verified binding and token rotation semantics. |
+| Provider attempt authorization/OAuth broker | CONTROL_API_LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MIN=300; CONTROL_API_OAUTH_BROKER_RL_PER_MIN=300 | Existing verified domain budgets; unverified attempt-authorization IP60 remains. |
+| Administrative GFS grants/shares/report | CONTROL_API_ADMIN_GFS_GRANTS_PER_MIN=150; CONTROL_API_ADMIN_GFS_SHARES_PER_MIN=150; CONTROL_API_ADMIN_GFS_LEGACY_GRANT_REPORT_PER_MIN=150 | Existing operator permissions; grants/read-write and shares keep their own families. |
+
+Code defaults and base/minikube ConfigMap literals must agree. A code-only
+increase can be defeated by an older deployment override. This document and
+changed manifests are source artifacts, not confirmation of a rollout.
+
+## SDK operation multiplicity
+
+A healthy static-secret attempt performs reissue, pre-Secret introspection and
+post-Secret introspection/redemption. At the old 120 operation allowance this
+guard admitted at most40 healthy attempts per minute. All three operations
+are retained. 1800 operations make600 healthy single-attempt prompts possible
+through this guard.
+
+Each sequential prompt also requests fresh capability proof, authorizes and
+closes its lifecycle. SDK-only closure totals six Control API requests; a
+runtime with separate provider/invocation status closure totals seven. 600
+healthy prompts therefore consume 3600/4200 general requests before bootstrap,
+other methods or retries. Both authenticated request gates default to 6000.
+Capability authority is not cached to reduce those charges.
+
+The three credential requests carry the Host runtime access JWT, including
+the requests forwarded by WRC. They are attributed to that verified Host or
+recipe, not to WRC’s internal service identity. Namespace equal to the Hosts
+namespace uses primary hostRefs[0]; other callers use namespace/recipeName.
+Both paths retain signature, audience, expiry and normalized binding checks.
+
+## Windows, headers and backend outages
+
+Authenticated edge counters and the PostgreSQL ledger use calendar minutes.
+A late first request no longer leaves an exhausted edge counter active after
+the database reset. Server-generated source-IP keys retain the previous
+first-hit abuse window. Always-counted calendar stores do not enable
+skipSuccessfulRequests or skipFailedRequests.
+
+The last enforcing ledger or process-memory counter owns draft7, draft6 and
+legacy X quota metadata. 429 includes rate_limited, a readable message,
+retryAfterSeconds and Retry-After. 503 unavailable removes stale quota metadata
+and retains its retry contract; an unavailable count is not zero usage.
+
+The existing PostgreSQL outage policy is unchanged. Process-memory counters
+are per process and use first-hit 60 s windows with bounded key cardinality.
+Their counters do not combine with PostgreSQL, and they are not a distributed
+quota. Outage recovery can therefore differ from normal ledger enforcement.
+
+## Naming and active accounting
+
+Shared infrastructure uses capability names. The previous admin_codex_read,
+admin_codex_write and codex_oauth_callback categories become subscription
+categories; legitimate vendor protocol identifiers remain specific.
+
+The namespace migration merges counts atomically inside the existing migration
+transaction and preserves windows/suffixes. An invoker trigger normalizes old
+writers during rolling updates and supported rollback. Existing bounded long
+key digests and the 512-byte boundary remain compatible. Do not clear counters
+to pass a smoke test. Retire the compatibility trigger only through a later
+reviewed migration after old writers and rollback support are retired.
+
+## Control UI recovery
+
+Only allowlisted safe subscription metadata uses browser-local reuse: TTL 30 s,
+maximum 128 entries, origin/principal/scope/session-epoch binding and a shared
+subscription-read cooldown. Ordinary Agent sections do not fetch optional
+subscription inventory; editor opening loads it when needed.
+
+At most one shared idempotent recovery GET proceeds after a denial deadline.
+Forced refresh respects the same pause. Authentication changes, 401 and
+cross-tab invalidation fence old awaits and clear identity-bound state.
+Aborting one subscriber does not cancel another subscriber’s shared request.
+SSR does not retain cached/deduplicated results across requests.
+
+Do not cache authentication decisions, general mutable reads, file contents,
+OAuth polling or credentials. Mutations and OAuth operations are not replayed
+automatically. Optional provider failures preserve independent Agent data,
+known summaries and drafts; unknown availability is not disabled availability.
+
+## Evidence and rollout boundary
+
+Unit/build, real PostgreSQL, local T0/T1/T2, visible browser and deployed
+receipts are distinct. Require actual selected tests, complete reports and
+producer exit 0; skipped physical tests do not establish correctness.
+
+For an authorized rollout, verify effective numeric owners, source/image
+revision, ledger migration ordering, old-writer drain, independent identities,
+forwarding policy and visible recovery. Keep upstream provider quotas, bytes,
+concurrency and unrelated authentication/minting safeguards separate.429 must
+remain possible under deliberate overload and must be understandable.
+
+References: issue #985; #433 owns broader edge-store/cardinality coordination;
+#438 owns changes to principal policy. Neither is silently replaced here.
diff --git a/docs/crds/workflowrecipe.md b/docs/crds/workflowrecipe.md
index d5aefdfcf..51899d793 100644
--- a/docs/crds/workflowrecipe.md
+++ b/docs/crds/workflowrecipe.md
@@ -1377,12 +1377,12 @@ Declaring this block forces an always-on `mcp-host`. Runtime enforcement is addi
| `promptBridge.maxOutputTokens` | integer | No | Min 1. Enforced as `max_tokens` on API-key providers (OpenAI, Claude, Z.AI, Bedrock, Google). `codex-subscription` cannot bind it: the ChatGPT wire rejects `max_output_tokens`, so there the ceiling only shapes the authorize request identity and the enforced bound is the contract limit of 16384. |
| `promptBridge.maxRequestsPerRun` | integer | No | Min 1. **Deprecated:** accepted but ignored; platform per-minute rate limits apply. Removal planned. |
| `promptBridge.maxConcurrentInvocations` | integer | No | Min 1. Runtime default 5. |
-| `promptBridge.maxInvocationsPerMinute` | integer | No | Min 1. Platform default 120 when unset (ENV `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN`, issue #348). |
+| `promptBridge.maxInvocationsPerMinute` | integer | No | Min 1. Platform default 600 when unset (ENV `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN`, issue #348). |
| `clientNotifications.allowedEventTypes` | string[] | **Yes**\† | 1–64 entries. |
| `clientNotifications.allowedTargetRefs` | string[] | No | Max 64. |
| `clientNotifications.allowedUserRefs` | boolean | No | Whether `userRef` targets are permitted. |
| `clientNotifications.maxNotificationsPerRun` | integer | No | Min 1. **Deprecated:** accepted but ignored; platform per-minute rate limits apply. Removal planned. |
-| `clientNotifications.maxNotificationsPerMinute` | integer | No | Min 1. Platform default 150 when unset (ENV `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN`, issue #348). |
+| `clientNotifications.maxNotificationsPerMinute` | integer | No | Min 1. Platform default 750 when unset (ENV `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN`, issue #348). |
| `allowedCallers` | string[] | No | Workload ids permitted to call the SDK. **Empty = all declared workloads.** |
| `idempotencyKeyPattern` | string | No | Regex; runtime default `^[a-zA-Z0-9_-]{1,128}$`. Must compile. |
diff --git a/docs/deploy/plugin-workload-sdk-upgrade.md b/docs/deploy/plugin-workload-sdk-upgrade.md
index ab3b378fa..15fe1564c 100644
--- a/docs/deploy/plugin-workload-sdk-upgrade.md
+++ b/docs/deploy/plugin-workload-sdk-upgrade.md
@@ -63,10 +63,11 @@ permanently trapped.
| ENV variable | Default | Limits |
|---|---|---|
-| `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN` | `150` | clientNotifications invocations per minute |
-| `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN` | `120` | promptBridge invocations per minute |
-| `CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN` | `600` | shared SDK request bucket per minute |
-| `CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN` | `600` | pre-authorization checks per minute |
+| `CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN` | `750` | clientNotifications invocations per minute |
+| `CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN` | `600` | promptBridge invocations per minute |
+| `CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN` | `3000` | authenticated shared SDK request bucket per minute |
+| `CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN` | `600` | anonymous or invalid-credential source-IP pre-auth checks per minute |
+| `CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN` | `3000` | verified Host principal pre-auth checks per minute |
The defaults above equal the code defaults in `control-api/src/config.ts` and
are registered in both `deploy/base/control-plane/configmaps.yaml` and the
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 2b9c3013c..c6c0eb7d4 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index c43d9d0b3..1086afb5c 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.405",
+ "version": "0.1.406",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/types.ts b/workflow-recipes/src/types.ts
index c31b83287..ddd7865d0 100644
--- a/workflow-recipes/src/types.ts
+++ b/workflow-recipes/src/types.ts
@@ -515,7 +515,7 @@ export interface PluginWorkloadSdkSpec {
maxRequestContentBytes?: number
/** Max parallel promptBridge calls per workload; default 5. */
maxConcurrentInvocations?: number
- /** Rate limit per workload; platform default 120 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348). */
+ /** Rate limit per workload; platform default 600 when unset (ENV CONTROL_API_PLUGIN_SDK_PROMPTBRIDGE_PER_MIN, issue #348). */
maxInvocationsPerMinute?: number
/** How long invocation results are queryable; default 24. */
resultRetentionHours?: number
@@ -530,7 +530,7 @@ export interface PluginWorkloadSdkSpec {
allowedUserRefs?: boolean
/** @deprecated Accepted for compatibility; no longer enforced (issue #348). Platform per-minute rate limits apply. */
maxNotificationsPerRun?: number
- /** Rate limit per workload; platform default 150 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348). */
+ /** Rate limit per workload; platform default 750 when unset (ENV CONTROL_API_PLUGIN_SDK_NOTIFICATIONS_PER_MIN, issue #348). */
maxNotificationsPerMinute?: number
/** Default 256. */
maxTitleBytes?: number