Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: release failed read recoveries and align the SDK verification window

Alfredo Lopez86f976e3165a
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 6792f55e7..177496fb9 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.716",
+ "version": "0.1.717",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.716",
+ "version": "0.1.717",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 8ee1b9f0e..fa9a4b96e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.716",
+ "version": "0.1.717",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
index 12562f581..717133232 100644
--- a/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
+++ b/control-api/src/middleware/pluginWorkloadSdkRateLimits.ts
@@ -62,14 +62,18 @@ export function pluginSdkPreauthAssignment(principal: string | null): {
* verified SDK allowance: invalid credentials still stop at the IP600 ceiling
* and valid callers behind a flooded IP keep passing until this budget, after
* which further tokens from that IP are denied without being verified.
+ * It shares the calendar minute of the gates it bounds, so it cannot stay
+ * exhausted after they reset. The key deliberately has no `:ip:` marker,
+ * which would keep CalendarMinuteRateLimitStore on a first-hit window.
*/
export function createPluginWorkloadSdkVerificationBudgetRateLimit(): RequestHandler {
return rateLimit({
windowMs: 60_000,
+ store: new CalendarMinuteRateLimitStore(),
limit: config.pluginSdkAuthenticatedPreauthRlPerMin,
skip: req => extractBearerToken(req) === '',
keyGenerator: req =>
- `plugin_workload_sdk_verification:ip:${ipKeyGenerator(req.ip || 'unknown')}`,
+ `plugin_workload_sdk_verification:source:${ipKeyGenerator(req.ip || 'unknown')}`,
standardHeaders: 'draft-8',
legacyHeaders: false,
message: { error: 'Too Many Requests', retryable: true },
diff --git a/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts b/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts
index 2d4aebade..21e8f6093 100644
--- a/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts
+++ b/control-api/test/pluginWorkloadSdkVerificationBudget.test.ts
@@ -98,6 +98,30 @@ describe('Plugin Workload SDK verification budget', () => {
expect(mcpHostVerificationsOf(valid)).toBe(1)
})
+ it('resets on the same calendar minute as the authenticated gate it bounds', async () => {
+ config.pluginSdkAuthenticatedPreauthRlPerMin = 2
+ config.pluginSdkPreauthRlPerMin = 600
+ vi.useFakeTimers({ toFake: ['Date'] })
+ try {
+ vi.setSystemTime(new Date('2026-01-01T00:00:50.000Z'))
+ const instance = preauthApp()
+ const valid = signedHost()
+ for (let i = 0; i < 2; i += 1) {
+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(200)
+ }
+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(429)
+ expect(mcpHostVerificationsOf(valid)).toBe(2)
+
+ // Ten seconds later the calendar minute has turned: the gate admits the
+ // principal again, and the budget in front of it must not still deny.
+ vi.setSystemTime(new Date('2026-01-01T00:01:00.500Z'))
+ await request(instance).get('/sdk-gate').set('Authorization', `Bearer ${valid}`).expect(200)
+ expect(mcpHostVerificationsOf(valid)).toBe(3)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('does not spend the budget on requests that present no bearer token', async () => {
config.pluginSdkAuthenticatedPreauthRlPerMin = 1
config.pluginSdkPreauthRlPerMin = 10
diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.ts
index e300ec4a8..294ebe51a 100644
--- a/control-ui/lib/__tests__/readRequestCache.test.ts
+++ b/control-ui/lib/__tests__/readRequestCache.test.ts
@@ -366,6 +366,55 @@ describe('bounded metadata read reuse', () => {
expect(fetchMock).toHaveBeenCalledTimes(2)
})
+ it('schedules one recovery for a later denial after a recovery was itself denied', async () => {
+ const path = '/api/v1/admin/connections'
+ const connections = { metadataRead: 'subscription-connections' } as const
+ fetchMock.mockResolvedValueOnce(throttled(12)).mockResolvedValueOnce(throttled(12))
+ await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })
+ await vi.advanceTimersByTimeAsync(12_000)
+ // The single recovery ran and was denied; it does not schedule another.
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+
+ fetchMock
+ .mockResolvedValueOnce(throttled(12))
+ .mockResolvedValueOnce(success({ connections: ['recovered'] }))
+ await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })
+ expect(fetchMock).toHaveBeenCalledTimes(3)
+ await vi.advanceTimersByTimeAsync(12_000)
+ expect(fetchMock).toHaveBeenCalledTimes(4)
+ expect(String(fetchMock.mock.calls[3][0])).toContain(path)
+ await expect(apiGet(path, {}, connections)).resolves.toEqual({ connections: ['recovered'] })
+ expect(fetchMock).toHaveBeenCalledTimes(4)
+ })
+
+ it('schedules a recovery for a remount after every earlier subscriber left', async () => {
+ const path = '/api/v1/admin/connections'
+ const firstMount = new AbortController()
+ fetchMock.mockResolvedValueOnce(throttled(12))
+ await expect(
+ apiGet(path, {}, { metadataRead: 'subscription-connections', signal: firstMount.signal })
+ ).rejects.toMatchObject({ status: 429 })
+ firstMount.abort()
+ await vi.advanceTimersByTimeAsync(24_000)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+
+ const secondMount = new AbortController()
+ fetchMock
+ .mockResolvedValueOnce(throttled(12))
+ .mockResolvedValueOnce(success({ connections: ['remounted'] }))
+ await expect(
+ apiGet(path, {}, { metadataRead: 'subscription-connections', signal: secondMount.signal })
+ ).rejects.toMatchObject({ status: 429 })
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(12_000)
+ expect(fetchMock).toHaveBeenCalledTimes(3)
+ await expect(apiGet(path, {}, { metadataRead: 'subscription-connections' })).resolves.toEqual({
+ connections: ['remounted'],
+ })
+ })
+
it('clears metadata on an epoch change and broadcasts only a non-sensitive invalidation', async () => {
const listener = new BroadcastChannel('control-ui-read-metadata-invalidation')
const received: unknown[] = []
diff --git a/control-ui/lib/readRequestCache.ts b/control-ui/lib/readRequestCache.ts
index 7cca1f4eb..f535015db 100644
--- a/control-ui/lib/readRequestCache.ts
+++ b/control-ui/lib/readRequestCache.ts
@@ -22,7 +22,7 @@ type RecoveryEntry = {
pending: Promise<void>
resolve: () => void
reject: (error: unknown) => void
- state: 'scheduled' | 'running' | 'failed'
+ state: 'scheduled' | 'running'
removeListeners: () => void
addSubscribers: (signals: Array<AbortSignal | undefined>) => void
}
@@ -200,9 +200,7 @@ function disposeRecovery(entry: RecoveryEntry): void {
export function getReadRequestRecovery(key: string): Promise<void> | undefined {
const recovery = recoveries.get(key)
- return recovery && recovery.state !== 'failed' && recovery.deadlineMs <= Date.now()
- ? recovery.pending
- : undefined
+ return recovery && recovery.deadlineMs <= Date.now() ? recovery.pending : undefined
}
export function reserveReadRequestRecovery(
@@ -235,9 +233,9 @@ export function reserveReadRequestRecovery(
const watchedSignals = new Set<AbortSignal>()
const onAbort = () => {
if (!Array.from(interests).every(signal => signal?.aborted)) return
- const entry = recoveries.get(key)
- if (!entry) return
- entry.state = 'failed'
+ // Release the reservation so a later mount's denial can schedule its own.
+ if (recoveries.get(key) !== entry) return
+ recoveries.delete(key)
disposeRecovery(entry)
}
const entry: RecoveryEntry = {
@@ -299,9 +297,12 @@ export function completeReadRequestRecovery(key: string): void {
}
export function failReadRequestRecovery(key: string, error: unknown): void {
+ // Only the running recovery reports its own failure. Its denial could not
+ // reserve another attempt, so releasing the entry keeps one recovery per
+ // denial while letting a later denial in this family schedule its own.
const recovery = recoveries.get(key)
- if (!recovery) return
- recovery.state = 'failed'
+ if (recovery?.state !== 'running') return
+ recoveries.delete(key)
recovery.removeListeners()
recovery.reject(error)
}
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index 236bd94a0..951b223b1 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.711",
+ "version": "0.1.712",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.711",
+ "version": "0.1.712",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 0895d2ba5..eedf13fd3 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.711",
+ "version": "0.1.712",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.md
index 8f8b7466d..291f1ab14 100644
--- a/docs/architecture/administrative-rate-limits.md
+++ b/docs/architecture/administrative-rate-limits.md
@@ -70,7 +70,8 @@ only requests that present a bearer token, uses
CONTROL_API_PLUGIN_SDK_AUTHENTICATED_PREAUTH_PER_MIN (6000) as its limit, and
denies further tokens from that IP without verifying them. Invalid credentials
still stop at IP600, and valid callers behind a flooded IP keep passing until
-the IP has spent 6000 verifications in the minute. Verified principals that
+the IP has spent 6000 verifications in the calendar minute, the same window
+the authenticated gate and the internal edge reset on. Verified principals that
share one source IP share this budget, and the internal SDK edge allowance
cannot exceed it from one IP.
@@ -150,7 +151,8 @@ subscription-read cooldown. Ordinary Agent sections do not fetch optional
subscription inventory; editor opening loads it when needed.
At most one shared idempotent recovery GET proceeds after a denial deadline.
-Forced refresh respects the same pause. Authentication changes, 401 and
+A denied recovery schedules no further attempt, but it releases its
+reservation, so the next denial in that read family schedules its own. Forced refresh respects the same pause. Authentication changes, 401 and
cross-tab invalidation fence old awaits and clear identity-bound state.
Aborting one subscriber does not cancel another subscriber’s shared request.
SSR does not retain cached/deduplicated results across requests.