Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: release failed read recoveries and align the SDK verification window
Alfredo Lopez86f976e3165a
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 6792f55e7..177496fb9 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.716",+ "version": "0.1.717","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.716",+ "version": "0.1.717","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 8ee1b9f0e..fa9a4b96e 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.716",+ "version": "0.1.717","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.tsindex 12562f581..717133232 100644--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts@@ -62,14 +62,18 @@ export function pluginSdkPreauthAssignment(principal: string | null): {* verified SDK allowance: invalid credentials still stop at the IP600 ceiling* and valid callers behind a flooded IP keep passing until this budget, after* which further tokens from that IP are denied without being verified.+ * It shares the calendar minute of the gates it bounds, so it cannot stay+ * exhausted after they reset. The key deliberately has no `:ip:` marker,+ * which would keep CalendarMinuteRateLimitStore on a first-hit window.*/export function createPluginWorkloadSdkVerificationBudgetRateLimit(): RequestHandler {return rateLimit({windowMs: 60_000,+ store: new CalendarMinuteRateLimitStore(),limit: config.pluginSdkAuthenticatedPreauthRlPerMin,skip: req => extractBearerToken(req) === '',keyGenerator: req =>- `plugin_workload_sdk_verification:ip:${ipKeyGenerator(req.ip || 'unknown')}`,+ `plugin_workload_sdk_verification:source:${ipKeyGenerator(req.ip || 'unknown')}`,standardHeaders: 'draft-8',legacyHeaders: false,message: { error: 'Too Many Requests', retryable: true },diff --git a/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts b/control-api/test/pluginWorkloadSdkVerificationBudget.test.tsindex 2d4aebade..21e8f6093 100644--- a/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts+++ b/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts@@ -98,6 +98,30 @@ describe('Plugin Workload SDK verification budget', () => {expect(mcpHostVerificationsOf(valid)).toBe(1)})+ it('resets on the same calendar minute as the authenticated gate it bounds', async () => {+ config.pluginSdkAuthenticatedPreauthRlPerMin = 2+ config.pluginSdkPreauthRlPerMin = 600+ vi.useFakeTimers({ toFake: ['Date'] })+ try {+ vi.setSystemTime(new Date('2026-01-01T00:00:50.000Z'))+ const instance = preauthApp()+ const valid = signedHost()+ for (let i = 0; i < 2; i += 1) {+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(200)+ }+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(429)+ expect(mcpHostVerificationsOf(valid)).toBe(2)++ // Ten seconds later the calendar minute has turned: the gate admits the+ // principal again, and the budget in front of it must not still deny.+ vi.setSystemTime(new Date('2026-01-01T00:01:00.500Z'))+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(200)+ expect(mcpHostVerificationsOf(valid)).toBe(3)+ } finally {+ vi.useRealTimers()+ }+ })+it('does not spend the budget on requests that present no bearer token', async () => {config.pluginSdkAuthenticatedPreauthRlPerMin = 1config.pluginSdkPreauthRlPerMin = 10diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.tsindex e300ec4a8..294ebe51a 100644--- a/control-ui/lib/__tests__/readRequestCache.test.ts+++ b/control-ui/lib/__tests__/readRequestCache.test.ts@@ -366,6 +366,55 @@ describe('bounded metadata read reuse', () => {expect(fetchMock).toHaveBeenCalledTimes(2)})+ it('schedules one recovery for a later denial after a recovery was itself denied', async () => {+ const path = '/api/v1/admin/connections'+ const connections = { metadataRead: 'subscription-connections' } as const+ fetchMock.mockResolvedValueOnce(throttled(12)).mockResolvedValueOnce(throttled(12))+ await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })+ await vi.advanceTimersByTimeAsync(12_000)+ // The single recovery ran and was denied; it does not schedule another.+ expect(fetchMock).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(60_000)+ expect(fetchMock).toHaveBeenCalledTimes(2)++ fetchMock+ .mockResolvedValueOnce(throttled(12))+ .mockResolvedValueOnce(success({ connections: ['recovered'] }))+ await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })+ expect(fetchMock).toHaveBeenCalledTimes(3)+ await vi.advanceTimersByTimeAsync(12_000)+ expect(fetchMock).toHaveBeenCalledTimes(4)+ expect(String(fetchMock.mock.calls[3][0])).toContain(path)+ await expect(apiGet(path, {}, connections)).resolves.toEqual({ connections: ['recovered'] })+ expect(fetchMock).toHaveBeenCalledTimes(4)+ })++ it('schedules a recovery for a remount after every earlier subscriber left', async () => {+ const path = '/api/v1/admin/connections'+ const firstMount = new AbortController()+ fetchMock.mockResolvedValueOnce(throttled(12))+ await expect(+ apiGet(path, {}, { metadataRead: 'subscription-connections', signal: firstMount.signal })+ ).rejects.toMatchObject({ status: 429 })+ firstMount.abort()+ await vi.advanceTimersByTimeAsync(24_000)+ expect(fetchMock).toHaveBeenCalledTimes(1)++ const secondMount = new AbortController()+ fetchMock+ .mockResolvedValueOnce(throttled(12))+ .mockResolvedValueOnce(success({ connections: ['remounted'] }))+ await expect(+ apiGet(path, {}, { metadataRead: 'subscription-connections', signal: secondMount.signal })+ ).rejects.toMatchObject({ status: 429 })+ expect(fetchMock).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(12_000)+ expect(fetchMock).toHaveBeenCalledTimes(3)+ await expect(apiGet(path, {}, { metadataRead: 'subscription-connections' })).resolves.toEqual({+ connections: ['remounted'],+ })+ })+it('clears metadata on an epoch change and broadcasts only a non-sensitive invalidation', async () => {const listener = new BroadcastChannel('control-ui-read-metadata-invalidation')const received: unknown[] = []diff --git a/control-ui/lib/readRequestCache.ts b/control-ui/lib/readRequestCache.tsindex 7cca1f4eb..f535015db 100644--- a/control-ui/lib/readRequestCache.ts+++ b/control-ui/lib/readRequestCache.ts@@ -22,7 +22,7 @@ type RecoveryEntry = {pending: Promise<void>resolve: () => voidreject: (error: unknown) => void- state: 'scheduled' | 'running' | 'failed'+ state: 'scheduled' | 'running'removeListeners: () => voidaddSubscribers: (signals: Array<AbortSignal | undefined>) => void}@@ -200,9 +200,7 @@ function disposeRecovery(entry: RecoveryEntry): void {export function getReadRequestRecovery(key: string): Promise<void> | undefined {const recovery = recoveries.get(key)- return recovery && recovery.state !== 'failed' && recovery.deadlineMs <= Date.now()- ? recovery.pending- : undefined+ return recovery && recovery.deadlineMs <= Date.now() ? recovery.pending : undefined}export function reserveReadRequestRecovery(@@ -235,9 +233,9 @@ export function reserveReadRequestRecovery(const watchedSignals = new Set<AbortSignal>()const onAbort = () => {if (!Array.from(interests).every(signal => signal?.aborted)) return- const entry = recoveries.get(key)- if (!entry) return- entry.state = 'failed'+ // Release the reservation so a later mount's denial can schedule its own.+ if (recoveries.get(key) !== entry) return+ recoveries.delete(key)disposeRecovery(entry)}const entry: RecoveryEntry = {@@ -299,9 +297,12 @@ export function completeReadRequestRecovery(key: string): void {}export function failReadRequestRecovery(key: string, error: unknown): void {+ // Only the running recovery reports its own failure. Its denial could not+ // reserve another attempt, so releasing the entry keeps one recovery per+ // denial while letting a later denial in this family schedule its own.const recovery = recoveries.get(key)- if (!recovery) return- recovery.state = 'failed'+ if (recovery?.state !== 'running') return+ recoveries.delete(key)recovery.removeListeners()recovery.reject(error)}diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 236bd94a0..951b223b1 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.711",+ "version": "0.1.712","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.711",+ "version": "0.1.712","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 0895d2ba5..eedf13fd3 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.711",+ "version": "0.1.712","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.mdindex 8f8b7466d..291f1ab14 100644--- a/docs/architecture/administrative-rate-limits.md+++ b/docs/architecture/administrative-rate-limits.md@@ -70,7 +70,8 @@ only requests that present a bearer token, usesCONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN (6000) as its limit, anddenies further tokens from that IP without verifying them. Invalid credentialsstill stop at IP600, and valid callers behind a flooded IP keep passing until-the IP has spent 6000 verifications in the minute. Verified principals that+the IP has spent 6000 verifications in the calendar minute, the same window+the authenticated gate and the internal edge reset on. Verified principals thatshare one source IP share this budget, and the internal SDK edge allowancecannot exceed it from one IP.@@ -150,7 +151,8 @@ subscription-read cooldown. Ordinary Agent sections do not fetch optionalsubscription inventory; editor opening loads it when needed.At most one shared idempotent recovery GET proceeds after a denial deadline.-Forced refresh respects the same pause. Authentication changes, 401 and+A denied recovery schedules no further attempt, but it releases its+reservation, so the next denial in that read family schedules its own. Forced refresh respects the same pause. Authentication changes, 401 andcross-tab invalidation fence old awaits and clear identity-bound state.Aborting one subscriber does not cancel another subscriber’s shared request.SSR does not retain cached/deduplicated results across requests.