Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(control-api): configure higher administrative subscription capacity
Alfredo Lopez03ebaa5f32c7
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 1676feae0..6ad274302 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.706",+ "version": "0.1.707","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.706",+ "version": "0.1.707","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 0a2d8963e..32cb8e0a1 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.706",+ "version": "0.1.707","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex 728232fd0..4cda6e62b 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -208,6 +208,9 @@ type Config = {workflowArtifactDownloadMaxBytes: number// Per-minute rate limits (PG-backed token buckets).approvalRlRequestPerMin: number+ adminSubscriptionReadPerMin: number+ adminSubscriptionWritePerMin: number+ subscriptionOAuthCallbackPerMin: numberapprovalRlRefreshPerMin: numberapprovalRlExternalPerMin: numberapprovalRlExternalEdgePerMin: number@@ -1001,6 +1004,18 @@ export const config: Config = {'CLERUM_ATTACHMENT_MAX_BYTES',50 * 1024 * 1024),+ adminSubscriptionReadPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN',+ 150+ ),+ adminSubscriptionWritePerMin: positiveIntegerFromEnv(+ 'CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN',+ 100+ ),+ subscriptionOAuthCallbackPerMin: positiveIntegerFromEnv(+ 'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',+ 100+ ),approvalRlRequestPerMin: Number(process.env.APPROVAL_RL_REQUEST_PER_MIN || 120),approvalRlRefreshPerMin: Number(process.env.APPROVAL_RL_REFRESH_PER_MIN || 20),approvalRlExternalPerMin: externalRateLimitConfig.operation,diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.tsindex 8480d1980..f1f2e5edf 100644--- a/control-api/src/routes/workflows/shared/rateLimit.ts+++ b/control-api/src/routes/workflows/shared/rateLimit.ts@@ -1,6 +1,7 @@import type { Request, Response } from 'express'import { ipKeyGenerator, rateLimit } from 'express-rate-limit'import { createHash } from 'node:crypto'+import { config } from '../../../config.js'import { rateLimitMiddleware } from '../../../middleware/rateLimitMiddleware.js'import { verifyAdminToken } from '../../../utils/auth/adminAuthToken.js'import { verifyExternalSessionToken } from '../../../utils/auth/externalSessionAuthToken.js'@@ -16,9 +17,6 @@ const WORKFLOW_GRANT_WRITE_PER_MINUTE = 20const WORKFLOW_ADMIN_READ_PER_MINUTE = 60const ADMIN_OUTPUTS_READ_PER_MINUTE = 30const WORKFLOW_TRIGGER_PER_MINUTE = 10-const ADMIN_CODEX_READ_PER_MINUTE = 30-const ADMIN_CODEX_WRITE_PER_MINUTE = 20-const CODEX_OAUTH_CALLBACK_PER_MINUTE = 20const LLM_PROVIDER_ATTEMPT_AUTHORIZE_PER_MINUTE = 60/**@@ -140,17 +138,17 @@ export function adminOutputsReadRateLimits() {}function adminCodexReadEdgeRateLimit() {- return createWorkflowEdgeRateLimit('admin_codex_read_edge', ADMIN_CODEX_READ_PER_MINUTE)+ return createWorkflowEdgeRateLimit('admin_codex_read_edge', config.adminSubscriptionReadPerMin)}function adminCodexWriteEdgeRateLimit() {- return createWorkflowEdgeRateLimit('admin_codex_write_edge', ADMIN_CODEX_WRITE_PER_MINUTE)+ return createWorkflowEdgeRateLimit('admin_codex_write_edge', config.adminSubscriptionWritePerMin)}function adminCodexReadRateLimit() {return rateLimitMiddleware({bucketType: 'admin_codex_read',- maxPerMinute: ADMIN_CODEX_READ_PER_MINUTE,+ maxPerMinute: config.adminSubscriptionReadPerMin,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_read'),onBackendUnavailable: 'process-memory',})@@ -159,7 +157,7 @@ function adminCodexReadRateLimit() {function adminCodexWriteRateLimit() {return rateLimitMiddleware({bucketType: 'admin_codex_write',- maxPerMinute: ADMIN_CODEX_WRITE_PER_MINUTE,+ maxPerMinute: config.adminSubscriptionWritePerMin,getBucketKey: hashedAdminWorkflowCredentialBucket('admin_codex_write'),onBackendUnavailable: 'process-memory',})@@ -186,7 +184,7 @@ export function codexOAuthCallbackRateLimits() {return [rateLimit({windowMs: 60_000,- limit: CODEX_OAUTH_CALLBACK_PER_MINUTE,+ limit: config.subscriptionOAuthCallbackPerMin,standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: codexOAuthCallbackBucketKey,@@ -194,7 +192,7 @@ export function codexOAuthCallbackRateLimits() {}),rateLimitMiddleware({bucketType: 'codex_oauth_callback',- maxPerMinute: CODEX_OAUTH_CALLBACK_PER_MINUTE,+ maxPerMinute: config.subscriptionOAuthCallbackPerMin,getBucketKey: codexOAuthCallbackBucketKey,onBackendUnavailable: 'process-memory',}),diff --git a/control-api/test/config.adminRateLimits.test.ts b/control-api/test/config.adminRateLimits.test.tsnew file mode 100644index 000000000..eb8a3caed--- /dev/null+++ b/control-api/test/config.adminRateLimits.test.ts@@ -0,0 +1,56 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++const KEYS = [+ 'CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN',+ 'CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN',+ 'CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN',+] as const++afterEach(() => {+ vi.unstubAllEnvs()+ vi.resetModules()+})++async function readConfig(overrides: Partial<Record<(typeof KEYS)[number], string>> = {}) {+ for (const key of KEYS) vi.stubEnv(key, overrides[key] ?? '')+ vi.resetModules()+ const { config } = await import('../src/config.js')+ // Keep failure diagnostics limited to the contract under test. The full+ // configuration also owns cryptographic material and must never be printed.+ return {+ adminSubscriptionReadPerMin: config.adminSubscriptionReadPerMin,+ adminSubscriptionWritePerMin: config.adminSubscriptionWritePerMin,+ subscriptionOAuthCallbackPerMin: config.subscriptionOAuthCallbackPerMin,+ }+}++describe('administrative subscription capacity', () => {+ it('defaults to five times the incident thresholds', async () => {+ const config = await readConfig()+ expect(config.adminSubscriptionReadPerMin).toBe(150)+ expect(config.adminSubscriptionWritePerMin).toBe(100)+ expect(config.subscriptionOAuthCallbackPerMin).toBe(100)+ })++ it('honors distinct explicit operation budgets', async () => {+ const config = await readConfig({+ CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN: '321',+ CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN: '123',+ CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN: '234',+ })+ expect(config.adminSubscriptionReadPerMin).toBe(321)+ expect(config.adminSubscriptionWritePerMin).toBe(123)+ expect(config.subscriptionOAuthCallbackPerMin).toBe(234)+ })++ for (const key of KEYS) {+ it.each(['0', '-1', '1.5', 'NaN', '9007199254740992'])(+ `rejects invalid ${key}=%s at boot`,+ async value => {+ await expect(readConfig({ [key]: value })).rejects.toThrow(+ `${key} must be a positive integer`+ )+ }+ )+ }+})diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.tsindex e05695b0f..1d6960666 100644--- a/control-api/test/workflows.shared.rateLimit.test.ts+++ b/control-api/test/workflows.shared.rateLimit.test.ts@@ -1,4 +1,4 @@-import { beforeEach, describe, expect, it, vi } from 'vitest'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import request from 'supertest'import { config } from '../src/config.js'@@ -69,6 +69,139 @@ function pgAllows() {}describe('routes/workflows/shared/rateLimit', () => {+ afterEach(() => {+ vi.useRealTimers()+ })++ function countRequests() {+ const counts = new Map<string, number>()+ mockCheckAndIncrement.mockReset()+ mockCheckAndIncrement.mockImplementation(async (key: string, limit: number) => {+ const windowStartMs = Math.floor(Date.now() / 60_000) * 60_000+ const identity = `${key}|${windowStartMs}`+ const count = (counts.get(identity) ?? 0) + 1+ counts.set(identity, count)+ return {+ allowed: count <= limit,+ backendAvailable: true,+ remaining: Math.max(0, limit - count),+ resetMs: windowStartMs + 60_000,+ windowStartMs,+ count,+ }+ })+ return counts+ }++ it('shares 150 subscription reads across provider routes and recovers at reset', async () => {+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(1_800_000_000_000)+ const counts = countRequests()+ const app = express()+ for (const path of ['/codex/connections', '/grok/connections', '/grok/models']) {+ app.get(path, ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ }+ const paths = ['/codex/connections', '/grok/connections', '/grok/models']+ for (let i = 0; i < 150; i++) {+ await request(app)+ .get(paths[i % paths.length])+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(204)+ }+ expect(counts.size).toBe(1)+ const denied = await request(app)+ .get(paths[0])+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(429)+ expect(denied.headers['x-ratelimit-limit']).toBe('150')+ expect(denied.headers['retry-after']).toBeDefined()+ vi.setSystemTime(1_800_000_060_000)+ const recovered = await request(app)+ .get(paths[0])+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(204)+ expect(recovered.headers['x-ratelimit-remaining']).toBe('149')+ })++ it('gives distinct verified sessions independent subscription allowances', async () => {+ countRequests()+ const app = express()+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ for (const session of ['signed-admin-a', 'signed-admin-b']) {+ for (let i = 0; i < 150; i++) {+ await request(app)+ .get('/connections')+ .set('Cookie', `control_ui_admin_session=${session}`)+ .expect(204)+ }+ }+ await request(app)+ .get('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(429)+ })++ it('uses a configured read budget for both edge and distributed gates', async () => {+ const original = config.adminSubscriptionReadPerMin+ config.adminSubscriptionReadPerMin = 3+ try {+ countRequests()+ const app = express()+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ for (let i = 0; i < 3; i++) {+ await request(app)+ .get('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(204)+ }+ const denied = await request(app)+ .get('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(429)+ expect(denied.headers['ratelimit-policy']).toBe('3;w=60')+ expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === 3)).toBe(true)+ } finally {+ config.adminSubscriptionReadPerMin = original+ }+ })++ it('allows 100 subscription writes without consuming the read quota', async () => {+ const counts = countRequests()+ const app = express()+ app.post('/connections', ...adminCodexWriteRateLimits(), (_req, res) => res.sendStatus(204))+ app.get('/connections', ...adminCodexReadRateLimits(), (_req, res) => res.sendStatus(204))+ for (let i = 0; i < 100; i++) {+ await request(app)+ .post('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(204)+ }+ await request(app)+ .post('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(429)+ await request(app)+ .get('/connections')+ .set('Cookie', 'control_ui_admin_session=signed-admin-a')+ .expect(204)+ expect(counts.size).toBe(2)+ })++ it('enforces the 100 callback budget with retry timing', async () => {+ countRequests()+ const app = express()+ app.get('/callback', ...codexOAuthCallbackRateLimits(), (_req, res) => res.sendStatus(204))+ for (let i = 0; i < 100; i++) {+ await request(app).get('/callback').query({ state: 'unit-callback-state' }).expect(204)+ }+ const denied = await request(app)+ .get('/callback')+ .query({ state: 'unit-callback-state' })+ .expect(429)+ expect(denied.headers['ratelimit-policy']).toBe('100;w=60')+ expect(denied.headers['retry-after']).toBeDefined()+ })+beforeEach(() => {mockVerifyAdminToken.mockImplementation((token: string) =>token.startsWith('signed-') ? signedClaims(token.slice('signed-'.length)) : null