Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): bound OAuth callback state rotation per source IP
Alfredo Lopezbf4efb354470
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 4c05778ab..5ad5a6c01 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.721",+ "version": "0.1.722","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.721",+ "version": "0.1.722","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 1b5e6ed95..68c864d42 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.721",+ "version": "0.1.722","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/workflows/shared/rateLimit.ts b/control-api/src/routes/workflows/shared/rateLimit.tsindex d679444a7..e53ccc20e 100644--- a/control-api/src/routes/workflows/shared/rateLimit.ts+++ b/control-api/src/routes/workflows/shared/rateLimit.ts@@ -17,8 +17,11 @@ import {import { CONTROL_UI_ADMIN_SESSION_COOKIE, readCookie } from '../../../utils/auth/sessionCookies.js'import { extractBearerToken } from '../../../utils/extractBearerToken.js'-// Invalid credentials still use the original source-IP abuse budgets. Raised-// administrative capacity applies only after the existing signature check.+// Invalid credentials still use the original source-IP abuse budgets, and raised+// administrative capacity applies only after the existing signature check. The+// unauthenticated subscription OAuth callback has no credential to verify: it+// selects its budget by whether a `state` is present (see+// subscriptionOAuthCallbackRateLimits).const UNVERIFIED_WORKFLOW_GRANT_READ_PER_MINUTE = 60const UNVERIFIED_WORKFLOW_GRANT_WRITE_PER_MINUTE = 20const UNVERIFIED_WORKFLOW_ADMIN_READ_PER_MINUTE = 60@@ -32,7 +35,11 @@ function hasVerifiedAdminCredential(req: Request): boolean {return verifiedAdminRateLimitIdentity(adminWorkflowRateLimitCredential(req)) !== null}-/** Reuse the existing enforcer without raising a distinct unauthenticated gate. */+/**+ * Reuse the existing enforcer with two budgets: `isVerified` selects the raised+ * one. For administrative families it is a signature check; for the OAuth+ * callback it only reports that a `state` is present.+ */function withUnverifiedIpBudget(options: RateLimitEnforcerOptions & { getBucketKey: (req: Request) => string | null },unverifiedMaxPerMinute: number,@@ -286,10 +293,31 @@ function subscriptionOAuthCallbackBucketKey(req: Request): string {return `subscription_oauth_callback:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`}+/**+ * Callback limiters, in order:+ * 1. A per-source-IP ceiling across every `state`, edge only, so rotating+ * `state` values cannot mint unbounded buckets from one address. It adds no+ * PostgreSQL key: a ledger-backed ceiling would add the write it bounds. The+ * key must not reuse `subscription_oauth_callback:ip:`, which is the+ * no-state bucket below.+ * 2. The per-`state` edge limiter (the no-state IP safeguard without one).+ * 3. The PostgreSQL gate shared across replicas.+ */export function subscriptionOAuthCallbackRateLimits() {const hasState = (req: Request) =>typeof req.query.state === 'string' && req.query.state.trim().length > 0return [+ rateLimit({+ windowMs: 60_000,+ store: new CalendarMinuteRateLimitStore(),+ limit: config.subscriptionOAuthCallbackPerMin,+ standardHeaders: 'draft-7',+ legacyHeaders: false,+ keyGenerator: req =>+ `subscription_oauth_callback_ip_ceiling:ip:${ipKeyGenerator(req.ip ?? 'unknown')}`,+ handler: (req, res) =>+ edgeRateLimitDenied('subscription_oauth_callback_ip_ceiling_edge', req, res),+ }),rateLimit({windowMs: 60_000,store: new CalendarMinuteRateLimitStore(),diff --git a/control-api/test/workflows.shared.rateLimit.test.ts b/control-api/test/workflows.shared.rateLimit.test.tsindex 966418740..6b7aed01a 100644--- a/control-api/test/workflows.shared.rateLimit.test.ts+++ b/control-api/test/workflows.shared.rateLimit.test.ts@@ -3,6 +3,7 @@ import express from 'express'import { createHash } from 'node:crypto'import request from 'supertest'import { config } from '../src/config.js'+import { rateLimitHitsTotal } from '../src/observability/metrics.js'import {adminOutputsReadRateLimits,adminSubscriptionReadRateLimits,@@ -239,23 +240,100 @@ describe('routes/workflows/shared/rateLimit', () => {expect(counts.size).toBe(2)})- it('enforces the 100 callback budget with retry timing', async () => {- countRequests()+ function callbackApp() {+ vi.mocked(rateLimitHitsTotal.inc).mockClear()+ const handled = { count: 0 }const app = express()- app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>+ app.set('trust proxy', 1)+ app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) => {+ handled.count += 1res.sendStatus(204)- )+ })+ return { app, handled }+ }++ function deniedCallbackBuckets(): unknown[] {+ return vi+ .mocked(rateLimitHitsTotal.inc)+ .mock.calls.filter(([labels]) => (labels as { result?: string }).result === 'denied')+ .map(([labels]) => (labels as { bucket_type?: string }).bucket_type)+ }++ it('enforces the 100 per-state callback budget across source addresses', async () => {+ countRequests()+ const { app, handled } = callbackApp()+ // Each request comes from a different address, so the per-IP ceiling never+ // binds and the denial below can only come from the per-state limiter.for (let i = 0; i < 100; i++) {- await request(app).get('/callback').query({ state: 'unit-callback-state' }).expect(204)+ await request(app)+ .get('/callback')+ .set('X-Forwarded-For', `198.51.100.${i + 1}`)+ .query({ state: 'unit-callback-state' })+ .expect(204)}const denied = await request(app).get('/callback')+ .set('X-Forwarded-For', '203.0.113.200').query({ state: 'unit-callback-state' }).expect(429)+ expect(handled.count).toBe(100)+ expect(deniedCallbackBuckets()).toEqual(['subscription_oauth_callback_edge'])expect(denied.headers['ratelimit-policy']).toBe('100;w=60')expect(denied.headers['retry-after']).toBeDefined()})+ it('bounds state rotation from one source address at the edge without a ledger key', async () => {+ countRequests()+ const { app, handled } = callbackApp()+ for (let i = 0; i < 100; i++) {+ await request(app)+ .get('/callback')+ .set('X-Forwarded-For', '198.51.100.7')+ .query({ state: `rotated-state-${i}` })+ .expect(204)+ }+ const denied = await request(app)+ .get('/callback')+ .set('X-Forwarded-For', '198.51.100.7')+ .query({ state: 'rotated-state-100' })+ .expect(429)+ expect(denied.body.code).toBe('rate_limited')+ expect(handled.count).toBe(100)+ expect(deniedCallbackBuckets()).toEqual(['subscription_oauth_callback_ip_ceiling_edge'])+ // Every admitted request reached the ledger, keyed by its state; the+ // ceiling key never did.+ const ledgerKeys = mockCheckAndIncrement.mock.calls.map(call => String(call[0]))+ expect(ledgerKeys).toHaveLength(100)+ expect(ledgerKeys.every(key => key.startsWith('subscription_oauth_callback:state:'))).toBe(true)++ // Another source address keeps its own ceiling.+ await request(app)+ .get('/callback')+ .set('X-Forwarded-For', '198.51.100.8')+ .query({ state: 'rotated-state-other-ip' })+ .expect(204)+ expect(handled.count).toBe(101)+ })++ it('counts a caller-prepended forwarding address against the real source address', async () => {+ countRequests()+ const { app, handled } = callbackApp()+ for (let i = 0; i < 100; i++) {+ await request(app)+ .get('/callback')+ .set('X-Forwarded-For', '198.51.100.9')+ .query({ state: `spoof-state-${i}` })+ .expect(204)+ }+ await request(app)+ .get('/callback')+ .set('X-Forwarded-For', '9.9.9.9, 198.51.100.9')+ .query({ state: 'spoof-state-100' })+ .expect(429)+ expect(handled.count).toBe(100)+ expect(deniedCallbackBuckets()).toEqual(['subscription_oauth_callback_ip_ceiling_edge'])+ })+it.each([['workflow grant reads', 300, workflowGrantReadRateLimits],['workflow grant writes', 100, workflowGrantWriteRateLimits],@@ -281,13 +359,16 @@ describe('routes/workflows/shared/rateLimit', () => {it('retains the 20-request callback IP safeguard when no state is supplied', async () => {countRequests()- const app = express()- app.get('/callback', ...subscriptionOAuthCallbackRateLimits(), (_req, res) =>- res.sendStatus(204)- )+ const { app, handled } = callbackApp()for (let i = 0; i < 20; i++) await request(app).get('/callback').expect(204)await request(app).get('/callback').expect(429)+ expect(handled.count).toBe(20)+ expect(deniedCallbackBuckets()).toEqual(['subscription_oauth_callback_edge'])expect(mockCheckAndIncrement.mock.calls.every(call => call[1] === 20)).toBe(true)+ // The no-state safeguard and the per-IP ceiling are separate buckets: the+ // same address can still complete a callback that carries a state.+ await request(app).get('/callback').query({ state: 'after-no-state' }).expect(204)+ expect(handled.count).toBe(21)})// The Postgres gate is the counter shared across replicas, so it must enforce@@ -570,8 +651,9 @@ describe('routes/workflows/shared/rateLimit', () => {expect(adminOutputsReadRateLimits()).toHaveLength(2)expect(adminSubscriptionReadRateLimits()).toHaveLength(2)expect(adminSubscriptionWriteRateLimits()).toHaveLength(2)- expect(subscriptionOAuthCallbackRateLimits()).toHaveLength(2)expect(llmProviderAttemptAuthorizeRateLimits()).toHaveLength(2)+ // The callback adds an edge-only per-IP ceiling ahead of the pair.+ expect(subscriptionOAuthCallbackRateLimits()).toHaveLength(3)})it('shouldSkipWorkflowGrantEdgeRateLimit skips anonymous callers', () => {diff --git a/control-ui/e2e/qa-recorder-admin-rate-limit-recovery.spec.ts b/control-ui/e2e/qa-recorder-admin-rate-limit-recovery.spec.tsindex 894f5e04d..49dc72cf4 100644--- a/control-ui/e2e/qa-recorder-admin-rate-limit-recovery.spec.ts+++ b/control-ui/e2e/qa-recorder-admin-rate-limit-recovery.spec.ts@@ -55,7 +55,9 @@ const METADATA_REUSE_MS = 30_000type AgentFixture = { secretName: string; modelName: string; draftModelName: string; quota: number }type OwnedAgent = { name: string; contextNames: Set<string>; hostCreated: boolean }-type MetadataRead = { path: string; status?: number }+// Times are taken when Playwright reports the browser's request and response,+// so recovery timing is measured from the 429's arrival to each reread's start.+type MetadataRead = { path: string; status?: number; startedAtMs: number; respondedAtMs?: number }function apiPath(url: string): string {return new URL(url).pathname.replace(/^\/control-api(?=\/)/, '')@@ -146,12 +148,15 @@ function observeReads(page: Page) {(path === CAPABILITIES ||/^\/api\/v1\/admin\/llm\/providers\/(?:codex|grok)-subscription(?:\/|$)/.test(path))) {- reads.set(request, { path })+ reads.set(request, { path, startedAtMs: Date.now() })}}const recordResponse = (response: Response) => {const read = reads.get(response.request())- if (read) read.status = response.status()+ if (read) {+ read.status = response.status()+ read.respondedAtMs = Date.now()+ }const path = apiPath(response.url())if (path.startsWith('/api/v1/admin/') && response.status() === 429) throttles.push(path)}@@ -733,7 +738,12 @@ test.describe('optional QA recorder: administrative rate-limit journeys', () =>expect(body.code).toBe('rate_limited')expect(body.retryAfterSeconds).toBe(retryAfter)expect(body.message).toMatch(/try again in \d+ seconds/i)- const retryAt = Date.now() + retryAfter * 1_000+ // Anchor the deadline on the 429's arrival, not on the moment every+ // initial response has settled: a slower sibling response would+ // otherwise push the expected deadline past a correct recovery.+ const limitedRead = coldSession.snapshot().find(read => read.status === 429)+ expect(limitedRead?.respondedAtMs, 'The 429 arrival time was recorded').toBeDefined()+ const retryAt = limitedRead!.respondedAtMs! + retryAfter * 1_000const dialog = page.getByRole('dialog', { name: 'Edit model & credentials', exact: true })const alert = dialog.getByRole('alert')await expect(alert).toContainText(/(?:try again|retry) in \d+ seconds/i)@@ -755,24 +765,29 @@ test.describe('optional QA recorder: administrative rate-limit journeys', () =>// The editor recovers on its own at the deadline supplied by the real// guard: the shared background recovery rereads the denied inventory and// the alert, with its Retry button, goes away without a click. Clicking- // Retry here would race that recovery. The client starts its deadline- // when the 429 arrives, slightly before retryAt, hence the tolerance.+ // Retry here would race that recovery. Each reread must start at or+ // after the deadline; the tolerance covers the delay between the+ // browser's events and Playwright reporting them.const readsBeforeDeadline = 3await expect(dialog.getByRole('alert')).toHaveCount(0, {timeout: retryAfter * 1_000 + 15_000,})- expect(- Date.now(),- 'The automatic recovery must not read before the Retry-After deadline'- ).toBeGreaterThanOrEqual(retryAt - 1_000)await expect(dialog.getByRole('status').filter({ hasText: 'Loading subscription options…' })).toHaveCount(0)+ const recoveryStarts = coldSession+ .snapshot()+ .slice(readsBeforeDeadline)+ .map(read => read.startedAtMs - retryAt)+ expect(recoveryStarts.length, 'Recovery rereads the denied inventory').toBeGreaterThan(0)+ expect(+ Math.min(...recoveryStarts),+ 'No automatic recovery read starts before the Retry-After deadline'+ ).toBeGreaterThanOrEqual(-1_000)await expect(dialog.getByLabel('Current model', { exact: true })).toContainText(fixture.draftModelName)const afterDeadline = coldSession.snapshot().slice(readsBeforeDeadline)- expect(afterDeadline.length, 'Recovery reads the denied inventory').toBeGreaterThan(0)expect(afterDeadline.map(read => read.status),'No read after the Retry-After deadline is throttled again'diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.tsindex 412c977f6..f1ff241fb 100644--- a/control-ui/lib/__tests__/readRequestCache.test.ts+++ b/control-ui/lib/__tests__/readRequestCache.test.ts@@ -118,7 +118,17 @@ describe('bounded metadata read reuse', () => {}expect(fetchMock).toHaveBeenCalledTimes(129)+ // The newest and the oldest retained entries are served from the cache, so+ // the bound evicts exactly one entry rather than caching nothing.fetchMock.mockClear()+ await expect(apiGet('/api/v1/admin/metadata/128', {}, metadataOptions)).resolves.toEqual({+ index: 128,+ })+ await expect(apiGet('/api/v1/admin/metadata/1', {}, metadataOptions)).resolves.toEqual({+ index: 1,+ })+ expect(fetchMock).not.toHaveBeenCalled()+fetchMock.mockResolvedValueOnce(success({ index: 'fresh-first' }))await expect(apiGet('/api/v1/admin/metadata/0', {}, metadataOptions)).resolves.toEqual({index: 'fresh-first',diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 52d8e326e..54fd0746a 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.720",+ "version": "0.1.721","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.720",+ "version": "0.1.721","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex d302541cf..686b19dca 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.720",+ "version": "0.1.721","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.mdindex 48c5d14ed..9538e6523 100644--- a/docs/architecture/administrative-rate-limits.md+++ b/docs/architecture/administrative-rate-limits.md@@ -40,14 +40,14 @@ lower overrides lower the effective workload capacity.| --- | --- | --- || Subscription reads | CONTROL_API_ADMIN_SUBSCRIPTION_READ_PER_MIN=150 | Same signed browser credential across providers/capabilities; aligned edge and ledger. || Subscription writes | CONTROL_API_ADMIN_SUBSCRIPTION_WRITE_PER_MIN=100 | Separate from reads; no automatic replay. |-| Subscription callback | CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN=100 | Per supplied state; no-state IP safeguard20. Presence is not state validation. |+| Subscription callback | CONTROL_API_SUBSCRIPTION_OAUTH_CALLBACK_PER_MIN=100 | Per supplied state (20 in `dev`); no-state IP safeguard 20. An edge-only ceiling of the same value per source IP per pod bounds `state` rotation; `dev` had no per-IP bound on callbacks that carry a state. Presence is not state validation. || Workflow administrative reads | CONTROL_API_ADMIN_WORKFLOW_READ_PER_MIN=300 | Existing verified administrator credential. || Workflow grant reads/writes | CONTROL_API_ADMIN_WORKFLOW_GRANT_READ_PER_MIN=300; CONTROL_API_ADMIN_WORKFLOW_GRANT_WRITE_PER_MIN=100 | Separate families and aligned verified edge/ledger. |-| Administrative workflow trigger | CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN=50 | Existing verified caller. External/MCP trigger contract remains10. |+| Administrative workflow trigger | CONTROL_API_ADMIN_WORKFLOW_TRIGGER_PER_MIN=50 | Existing verified caller. External/MCP trigger contract remains 10. || Outputs reads | CONTROL_API_ADMIN_OUTPUTS_READ_PER_MIN=150 | Existing administrator binding. || Host wake | CONTROL_API_HOST_WAKE_RL_PER_MIN=150 | Verified Host reference after authentication and Host match. || Registry keys/grants/status/connect/recovery | CONTROL_API_ADMIN_REGISTRY_KEYS_PER_MIN=150; GRANTS=150; CONNECT_STATUS=150; CONNECT_REQUEST=15; CONNECT_RECOVERY=50 | Separate authenticated administrator-subject families; each suffix uses the CONTROL_API_ADMIN_REGISTRY_ prefix. |-| Connector deletion | CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN=300; CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN=150 | PostgreSQL150 controls the authenticated allowance. Authorization is preserved. |+| Connector deletion | CONTROL_API_ADMIN_CONNECTOR_DELETE_EDGE_PER_MIN=300; CONTROL_API_ADMIN_CONNECTOR_DELETE_PER_MIN=150 | PostgreSQL 150 controls the authenticated allowance. Authorization is preserved. || SDK admin/internal | CONTROL_API_PLUGIN_SDK_ADMIN_PER_MIN=600; CONTROL_API_PLUGIN_SDK_INTERNAL_PER_MIN=600 | Verified principal; internal signed iss/sub separated from invalid/anonymous IP600. || SDK authenticated request/preauth | CONTROL_API_PLUGIN_SDK_REQUEST_BUCKET_PER_MIN=6000; CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN=6000 | Verified runtime Host/recipe; aligned calendar minute. || SDK anonymous preauth | CONTROL_API_PLUGIN_SDK_PREAUTH_PER_MIN=600 | Source-IP first-hit backstop; a valid attributed runtime JWT does not consume it. |@@ -89,8 +89,8 @@ sharing its address.A healthy static-secret attempt performs reissue, pre-Secret introspection andpost-Secret introspection/redemption. At the old 120 operation allowance this-guard admitted at most40 healthy attempts per minute. All three operations-are retained. 1800 operations make600 healthy single-attempt prompts possible+guard admitted at most 40 healthy attempts per minute. All three operations+are retained. 1800 operations make 600 healthy single-attempt prompts possiblethrough this guard.Each sequential prompt also requests fresh capability proof, authorizes and