Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(control-api): configure higher administrative subscription capacity

Alfredo Lopez03ebaa5f32c7
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 1676feae0..6ad274302 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.706",
+ "version": "0.1.707",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.706",
+ "version": "0.1.707",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 0a2d8963e..32cb8e0a1 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.706",
+ "version": "0.1.707",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index 728232fd0..4cda6e62b 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -208,6 +208,9 @@ type Config = {
workflowArtifactDownloadMaxBytes: number
// Per-minute rate limits (PG-backed token buckets).
approvalRlRequestPerMin: number
+ adminSubscriptionReadPerMin: number
+ adminSubscriptionWritePerMin: number
+ subscriptionOAuthCallbackPerMin: number
approvalRlRefreshPerMin: number
approvalRlExternalPerMin: number
approvalRlExternalEdgePerMin: number
@@ -1001,6 +1004,18 @@ export const config: Config = {
'CLERUM_ATTACHMENT_MAX_BYTES',
50 * 1024 * 1024
),
+ adminSubscriptionReadPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN',
+ 150
+ ),
+ adminSubscriptionWritePerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN',
+ 100
+ ),
+ subscriptionOAuthCallbackPerMin: positiveIntegerFromEnv(
+ 'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',
+ 100
+ ),
approvalRlRequestPerMin: Number(process.env.APPROVAL_RL_REQUEST_PER_MIN || 120),
approvalRlRefreshPerMin: Number(process.env.APPROVAL_RL_REFRESH_PER_MIN || 20),
approvalRlExternalPerMin: externalRateLimitConfig.operation,
diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.ts
index 8480d1980..f1f2e5edf 100644
--- a/control-api/src/routes/workflows/shared/rateLimit.ts
+++ b/control-api/src/routes/workflows/shared/rateLimit.ts
@@ -1,6 +1,7 @@
import type { Request, Response } from 'express'
import { ipKeyGenerator, rateLimit } from 'express-rate-limit'
import { createHash } from 'node:crypto'
+import { config } from '../../../config.js'
import { rateLimitMiddleware } from '../../../middleware/rateLimitMiddleware.js'
import { verifyAdminToken } from '../../../utils/auth/adminAuthToken.js'
import { verifyExternalSessionToken } from '../../../utils/auth/externalSessionAuthToken.js'
@@ -16,9 +17,6 @@ const WORKFLOW_GRANT_WRITE_PER_MINUTE = 20
const WORKFLOW_ADMIN_READ_PER_MINUTE = 60
const ADMIN_OUTPUTS_READ_PER_MINUTE = 30
const WORKFLOW_TRIGGER_PER_MINUTE = 10
-const ADMIN_CODEX_READ_PER_MINUTE = 30
-const ADMIN_CODEX_WRITE_PER_MINUTE = 20
-const CODEX_OAUTH_CALLBACK_PER_MINUTE = 20
const LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE = 60
/**
@@ -140,17 +138,17 @@ export function adminOutputsReadRateLimits() {
}
function adminCodexReadEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('admin_codex_read_edge', ADMIN_CODEX_READ_PER_MINUTE)
+ return createWorkflowEdgeRateLimit('admin_codex_read_edge', config.adminSubscriptionReadPerMin)
}
function adminCodexWriteEdgeRateLimit() {
- return createWorkflowEdgeRateLimit('admin_codex_write_edge', ADMIN_CODEX_WRITE_PER_MINUTE)
+ return createWorkflowEdgeRateLimit('admin_codex_write_edge', config.adminSubscriptionWritePerMin)
}
function adminCodexReadRateLimit() {
return rateLimitMiddleware({
bucketType: 'admin_codex_read',
- maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,
+ maxPerMinute: config.adminSubscriptionReadPerMin,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),
onBackendUnavailable: 'process-memory',
})
@@ -159,7 +157,7 @@ function adminCodexReadRateLimit() {
function adminCodexWriteRateLimit() {
return rateLimitMiddleware({
bucketType: 'admin_codex_write',
- maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,
+ maxPerMinute: config.adminSubscriptionWritePerMin,
getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),
onBackendUnavailable: 'process-memory',
})
@@ -186,7 +184,7 @@ export function codexOAuthCallbackRateLimits() {
return [
rateLimit({
windowMs: 60_000,
- limit: CODEX_OAUTH_CALLBACK_PER_MINUTE,
+ limit: config.subscriptionOAuthCallbackPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: codexOAuthCallbackBucketKey,
@@ -194,7 +192,7 @@ export function codexOAuthCallbackRateLimits() {
}),
rateLimitMiddleware({
bucketType: 'codex_oauth_callback',
- maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,
+ maxPerMinute: config.subscriptionOAuthCallbackPerMin,
getBucketKey: codexOAuthCallbackBucketKey,
onBackendUnavailable: 'process-memory',
}),
diff --git a/control-api/test/config.adminRateLimits.test.ts b/control-api/test/config.adminRateLimits.test.ts
new file mode 100644
index 000000000..eb8a3caed
--- /dev/null
+++ b/control-api/test/config.adminRateLimits.test.ts
@@ -0,0 +1,56 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const KEYS = [
+ 'CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN',
+ 'CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN',
+ 'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',
+] as const
+
+afterEach(() => {
+ vi.unstubAllEnvs()
+ vi.resetModules()
+})
+
+async function readConfig(overrides: Partial<Record<(typeof KEYS)[number], string>> = {}) {
+ for (const key of KEYS) vi.stubEnv(key, overrides[key] ?? '')
+ vi.resetModules()
+ const { config } = await import('../src/config.js')
+ // Keep failure diagnostics limited to the contract under test. The full
+ // configuration also owns cryptographic material and must never be printed.
+ return {
+ adminSubscriptionReadPerMin: config.adminSubscriptionReadPerMin,
+ adminSubscriptionWritePerMin: config.adminSubscriptionWritePerMin,
+ subscriptionOAuthCallbackPerMin: config.subscriptionOAuthCallbackPerMin,
+ }
+}
+
+describe('administrative subscription capacity', () => {
+ it('defaults to five times the incident thresholds', async () => {
+ const config = await readConfig()
+ expect(config.adminSubscriptionReadPerMin).toBe(150)
+ expect(config.adminSubscriptionWritePerMin).toBe(100)
+ expect(config.subscriptionOAuthCallbackPerMin).toBe(100)
+ })
+
+ it('honors distinct explicit operation budgets', async () => {
+ const config = await readConfig({
+ CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN: '321',
+ CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN: '123',
+ CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN: '234',
+ })
+ expect(config.adminSubscriptionReadPerMin).toBe(321)
+ expect(config.adminSubscriptionWritePerMin).toBe(123)
+ expect(config.subscriptionOAuthCallbackPerMin).toBe(234)
+ })
+
+ for (const key of KEYS) {
+ it.each(['0', '-1', '1.5', 'NaN', '9007199254740992'])(
+ `rejects invalid ${key}=%s at boot`,
+ async value => {
+ await expect(readConfig({ [key]: value })).rejects.toThrow(
+ `${key} must be a positive integer`
+ )
+ }
+ )
+ }
+})
diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.ts
index e05695b0f..1d6960666 100644
--- a/control-api/test/workflows.shared.rateLimit.test.ts
+++ b/control-api/test/workflows.shared.rateLimit.test.ts
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import request from 'supertest'
import { config } from '../src/config.js'
@@ -69,6 +69,139 @@ function pgAllows() {
}
describe('routes/workflows/shared/rateLimit', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ function countRequests() {
+ const counts = new Map<string, number>()
+ mockCheckAndIncrement.mockReset()
+ mockCheckAndIncrement.mockImplementation(async (key: string, limit: number) => {
+ const windowStartMs = Math.floor(Date.now() / 60_000) * 60_000
+ const identity = `${key}|${windowStartMs}`
+ const count = (counts.get(identity) ?? 0) + 1
+ counts.set(identity, count)
+ return {
+ allowed: count <= limit,
+ backendAvailable: true,
+ remaining: Math.max(0, limit - count),
+ resetMs: windowStartMs + 60_000,
+ windowStartMs,
+ count,
+ }
+ })
+ return counts
+ }
+
+ it('shares 150 subscription reads across provider routes and recovers at reset', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] })
+ vi.setSystemTime(1_800_000_000_000)
+ const counts = countRequests()
+ const app = express()
+ for (const path of ['/codex/connections', '/grok/connections', '/grok/models']) {
+ app.get(path, ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ }
+ const paths = ['/codex/connections', '/grok/connections', '/grok/models']
+ for (let i = 0; i < 150; i++) {
+ await request(app)
+ .get(paths[i % paths.length])
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(204)
+ }
+ expect(counts.size).toBe(1)
+ const denied = await request(app)
+ .get(paths[0])
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(429)
+ expect(denied.headers['x-ratelimit-limit']).toBe('150')
+ expect(denied.headers['retry-after']).toBeDefined()
+ vi.setSystemTime(1_800_000_060_000)
+ const recovered = await request(app)
+ .get(paths[0])
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(204)
+ expect(recovered.headers['x-ratelimit-remaining']).toBe('149')
+ })
+
+ it('gives distinct verified sessions independent subscription allowances', async () => {
+ countRequests()
+ const app = express()
+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ for (const session of ['signed-admin-a', 'signed-admin-b']) {
+ for (let i = 0; i < 150; i++) {
+ await request(app)
+ .get('/connections')
+ .set('Cookie', `control_ui_admin_session=${session}`)
+ .expect(204)
+ }
+ }
+ await request(app)
+ .get('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(429)
+ })
+
+ it('uses a configured read budget for both edge and distributed gates', async () => {
+ const original = config.adminSubscriptionReadPerMin
+ config.adminSubscriptionReadPerMin = 3
+ try {
+ countRequests()
+ const app = express()
+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ for (let i = 0; i < 3; i++) {
+ await request(app)
+ .get('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(204)
+ }
+ const denied = await request(app)
+ .get('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(429)
+ expect(denied.headers['ratelimit-policy']).toBe('3;w=60')
+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === 3)).toBe(true)
+ } finally {
+ config.adminSubscriptionReadPerMin = original
+ }
+ })
+
+ it('allows 100 subscription writes without consuming the read quota', async () => {
+ const counts = countRequests()
+ const app = express()
+ app.post('/connections', ...adminCodexWriteRateLimits(), (_req, res) => res.sendStatus(204))
+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))
+ for (let i = 0; i < 100; i++) {
+ await request(app)
+ .post('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(204)
+ }
+ await request(app)
+ .post('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(429)
+ await request(app)
+ .get('/connections')
+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')
+ .expect(204)
+ expect(counts.size).toBe(2)
+ })
+
+ it('enforces the 100 callback budget with retry timing', async () => {
+ countRequests()
+ const app = express()
+ app.get('/callback', ...codexOAuthCallbackRateLimits(), (_req, res) => res.sendStatus(204))
+ for (let i = 0; i < 100; i++) {
+ await request(app).get('/callback').query({ state: 'unit-callback-state' }).expect(204)
+ }
+ const denied = await request(app)
+ .get('/callback')
+ .query({ state: 'unit-callback-state' })
+ .expect(429)
+ expect(denied.headers['ratelimit-policy']).toBe('100;w=60')
+ expect(denied.headers['retry-after']).toBeDefined()
+ })
+
beforeEach(() => {
mockVerifyAdminToken.mockImplementation((token: string) =>
token.startsWith('signed-') ? signedClaims(token.slice('signed-'.length)) : null