Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-ui): keep member results and automatic recovery across family denials

Alfredo Lopez25247386e87f
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index aa6d7ce06..6094a5ab6 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.718",
+ "version": "0.1.719",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.718",
+ "version": "0.1.719",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index ad822f2bb..c12af00a3 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.718",
+ "version": "0.1.719",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.ts b/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.ts
new file mode 100644
index 000000000..f83e459df
--- /dev/null
+++ b/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, it, vi } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+
+vi.mock('../src/services/rateLimiterService.js', () => ({
+ checkAndIncrement: vi.fn(async (_key: string, maxPerMinute: number) => ({
+ allowed: true,
+ remaining: maxPerMinute - 1,
+ resetMs: Date.now() + 60_000,
+ windowStartMs: Date.now(),
+ count: 1,
+ backendAvailable: true,
+ })),
+}))
+
+function createGateway() {
+ return {
+ deleteSecret: vi.fn(async (name: string, namespace?: string) => ({
+ name,
+ namespace: namespace || 'mcp-server',
+ deleted: true as const,
+ })),
+ getSecret: vi.fn(async (name: string, namespace?: string) => ({
+ metadata: { name, namespace, uid: `uid-${name}`, resourceVersion: '1', labels: {} },
+ data: {},
+ })),
+ listResource: vi.fn(async () => [] as unknown[]),
+ }
+}
+
+function createRollbackPermitStore() {
+ return {
+ issue: vi.fn(async () => undefined),
+ claim: vi.fn(async () => null),
+ release: vi.fn(async () => undefined),
+ finalize: vi.fn(async () => undefined),
+ }
+}
+
+describe('DELETE /admin/mcp-secrets/:name edge wiring', () => {
+ it('mounts the calendar-minute edge limiter in front of the delete handler', async () => {
+ // The route's limiter is built when the module loads, so lower its limit
+ // before importing the router.
+ vi.resetModules()
+ const { config } = await import('../src/config.js')
+ config.adminConnectorDeleteEdgePerMin = 2
+ const { checkAndIncrement } = await import('../src/services/rateLimiterService.js')
+ const { createAdminSecretsRouter } = await import('../src/routes/admin/secrets.js')
+ const gateway = createGateway()
+ const app = express()
+ app.use(express.json())
+ app.use((req, _res, next) => {
+ ;(req as express.Request & { adminAuth?: { sub: string; jti: string } }).adminAuth = {
+ sub: 'admin-wiring',
+ jti: 'admin-wiring-jti',
+ }
+ next()
+ })
+ app.use(createAdminSecretsRouter(gateway as never, createRollbackPermitStore() as never))
+
+ const send = () =>
+ request(app)
+ .delete('/admin/mcp-secrets/wiring-creds')
+ .send({ uid: 'uid-wiring-creds', resourceVersion: '1' })
+ await send().expect(200)
+ await send().expect(200)
+ const denied = await send().expect(429)
+
+ // Witness: the two admitted deletes reached the ledger and the handler; the
+ // third was refused by the edge before either.
+ expect(gateway.deleteSecret).toHaveBeenCalledTimes(2)
+ expect(vi.mocked(checkAndIncrement)).toHaveBeenCalledTimes(2)
+ expect(denied.headers['ratelimit-policy']).toBe('2;w=60')
+ })
+})
diff --git a/control-ui/app/hosts/[name]/page.tsx b/control-ui/app/hosts/[name]/page.tsx
index 0c25f88a6..4f67825c5 100644
--- a/control-ui/app/hosts/[name]/page.tsx
+++ b/control-ui/app/hosts/[name]/page.tsx
@@ -36,6 +36,7 @@ import {
updateContext,
} from '../../../lib/api'
import type { ContextResource, ContextSpec, HostSecretResource } from '../../../lib/api'
+import type { ApiRequestError } from '../../../lib/api.types'
import {
CODEX_UNASSIGNED_CONNECTION_KEY,
type CodexSubscriptionConnectionView,
@@ -78,6 +79,7 @@ import {
validateLlmPolicy,
} from '../../../lib/llm'
import { credentialSelectValue, parseCredentialSelect } from '../../../lib/llmCredentialSelect'
+import { scheduleReadRequestRetry } from '../../../lib/readRequestCache'
import type { HostTab } from './types'
const TAB_LABELS: Record<HostTab, string> = {
@@ -269,6 +271,9 @@ export default function HostDetailsPage() {
const [subscriptionInventoryError, setSubscriptionInventoryError] = useState('')
const [subscriptionInventoryLoading, setSubscriptionInventoryLoading] = useState(false)
const [inventoryRetryNonce, setInventoryRetryNonce] = useState(0)
+ const [subscriptionInventoryThrottle, setSubscriptionInventoryThrottle] =
+ useState<ApiRequestError | null>(null)
+ const inventoryAutoRetryUsedRef = useRef(false)
const catalogForEditor = useMemo(() => {
if (providerDraft === 'codex-subscription') {
const others = allowedCatalog.filter(row => row.provider !== 'codex-subscription')
@@ -426,6 +431,7 @@ export default function HostDetailsPage() {
useEffect(() => {
if (!editingModel || !subscriptionCapabilities.capabilities) {
setSubscriptionInventoryError('')
+ setSubscriptionInventoryThrottle(null)
setSubscriptionInventoryLoading(false)
return
}
@@ -447,11 +453,15 @@ export default function HostDetailsPage() {
setCodexConnections(codexResult.value)
setCodexInventoryLoaded(codexEnabled)
}
+ if (codexResult.status === 'fulfilled' && grokResult?.status !== 'rejected') {
+ inventoryAutoRetryUsedRef.current = false
+ }
if (grokResult?.status === 'fulfilled') {
setGrokConnections(grokResult.value)
setGrokInventoryLoaded(true)
}
setSubscriptionInventoryError('')
+ setSubscriptionInventoryThrottle(null)
const failure = [codexResult, grokResult].find(
result => result?.status === 'rejected' && !isDisabledCapabilityError(result.reason)
)
@@ -460,6 +470,8 @@ export default function HostDetailsPage() {
.catch(err => {
if (controller.signal.aborted || isDisabledCapabilityError(err)) return
setSubscriptionInventoryError(subscriptionInventoryMessage(err, 'subscription'))
+ const denied = err as ApiRequestError
+ setSubscriptionInventoryThrottle(denied?.status === 429 ? denied : null)
})
.finally(() => {
if (!controller.signal.aborted) setSubscriptionInventoryLoading(false)
@@ -473,6 +485,19 @@ export default function HostDetailsPage() {
subscriptionCapabilities.capabilities,
])
+ // The family's shared recovery rereads a denied inventory once Retry-After
+ // passes. Re-run the load once at that time: it waits for the recovery and
+ // shows its cached result without a click or a second read. A denial on that
+ // run leaves the visible Retry; a success or a manual Retry restores the
+ // automatic attempt for the next denial.
+ useEffect(() => {
+ if (!subscriptionInventoryThrottle || inventoryAutoRetryUsedRef.current) return
+ return scheduleReadRequestRetry(subscriptionInventoryThrottle, () => {
+ inventoryAutoRetryUsedRef.current = true
+ setInventoryRetryNonce(value => value + 1)
+ })
+ }, [subscriptionInventoryThrottle])
+
useEffect(() => {
if (!editingModel) return
if (!connectionRefDraft.trim() || connectionRefDraft === CODEX_UNASSIGNED_CONNECTION_KEY) {
@@ -1519,7 +1544,10 @@ export default function HostDetailsPage() {
<button
type="button"
className="cu-btn cu-btn--ghost cu-btn--sm"
- onClick={() => setInventoryRetryNonce(value => value + 1)}
+ onClick={() => {
+ inventoryAutoRetryUsedRef.current = false
+ setInventoryRetryNonce(value => value + 1)
+ }}
>
Retry
</button>
diff --git a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
index f66f9df10..1e1b4fc61 100644
--- a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
+++ b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx
@@ -19,9 +19,18 @@ import {
revokeGrokSubscription,
startGrokDeviceConnect,
} from '@lib/grokSubscription'
+import { __resetReadRequestCacheForTests, setReadRequestPrincipal } from '@lib/readRequestCache'
import { CodexSubscriptionHub } from '../CodexSubscriptionHub'
import { ToastProvider } from '../Toast'
+// Capability consumers load only for a principal confirmed by /me (AuthContext).
+beforeEach(() => {
+ setReadRequestPrincipal('admin-one', 'admin')
+})
+afterEach(() => {
+ __resetReadRequestCacheForTests()
+})
+
const confirmMock = vi.fn()
const capabilityProbes = vi.hoisted(() => ({
diff --git a/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx b/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx
index ed2293451..e6f86358d 100644
--- a/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx
+++ b/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx
@@ -19,8 +19,20 @@ import {
listGrokConnectionModels,
listGrokSubscriptionConnections,
} from '../../lib/grokSubscription'
+import {
+ __resetReadRequestCacheForTests,
+ setReadRequestPrincipal,
+} from '../../lib/readRequestCache'
import { ToastProvider } from '../Toast'
+// Capability consumers load only for a principal confirmed by /me (AuthContext).
+beforeEach(() => {
+ setReadRequestPrincipal('admin-one', 'admin')
+})
+afterEach(() => {
+ __resetReadRequestCacheForTests()
+})
+
const replaceMock = vi.fn()
const pushMock = vi.fn()
let mockParams: { name: string; tab?: string } = { name: 'foo' }
diff --git a/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx b/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx
index 339187c43..dab6c2ea8 100644
--- a/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx
+++ b/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx
@@ -278,7 +278,7 @@ describe('HostDetailsPage optional subscription throttling', () => {
expect(api.apiSend).not.toHaveBeenCalled()
})
- it('answers the visible Retry from the background recovery without spending another read in its window', async () => {
+ it('clears the error once the background recovery rereads the inventory, without a click or another read', async () => {
vi.useFakeTimers()
// The server's shared subscription-read quota: two reads per twelve-second window.
const windowMs = 12_000
@@ -316,20 +316,17 @@ describe('HostDetailsPage optional subscription throttling', () => {
expect(alert).toHaveTextContent('Try again in 12 seconds.')
expect(reads.map(read => read.status)).toEqual([200, 200, 429])
- // The shared recovery reads the denied inventory once the deadline passes.
+ // The shared recovery reads the denied inventory once the deadline passes,
+ // and the editor shows that result without a click or a read of its own.
await act(async () => {
await vi.advanceTimersByTimeAsync(windowMs)
})
- const afterDeadline = () => reads.filter(read => read.window >= 1)
- expect(afterDeadline()).toHaveLength(1)
- expect(afterDeadline()[0]?.status).toBe(200)
-
- fireEvent.click(within(alert).getByRole('button', { name: 'Retry' }))
await flush()
- // Witness: the visible error cleared through the Retry.
+ const afterDeadline = reads.filter(read => read.window >= 1)
+ expect(afterDeadline).toHaveLength(1)
+ expect(afterDeadline[0]?.status).toBe(200)
expect(within(dialog).queryByRole('alert')).toBeNull()
- expect(afterDeadline().map(read => read.status)).not.toContain(429)
- expect(afterDeadline().length).toBeLessThanOrEqual(2)
+ expect(screen.queryByText('Loading subscription options…')).toBeNull()
expect(api.apiSend).not.toHaveBeenCalled()
})
diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.ts
index bc412e28b..47253ae10 100644
--- a/control-ui/lib/__tests__/readRequestCache.test.ts
+++ b/control-ui/lib/__tests__/readRequestCache.test.ts
@@ -13,8 +13,13 @@ import {
__resetReadRequestCacheForTests,
getReadRequestCacheEntry,
getReadRequestPrincipal,
+ getReadRequestSessionIdentity,
invalidateReadRequestCache,
+ invalidateReadRequestCacheEntry,
+ reserveReadRequestRecovery,
+ scheduleReadRequestRetry,
setReadRequestCacheEntry,
+ setReadRequestCooldown,
} from '../readRequestCache'
const metadataOptions = { metadataRead: 'subscription-capabilities' } as const
@@ -432,6 +437,198 @@ describe('bounded metadata read reuse', () => {
expect(fetchMock).toHaveBeenCalledOnce()
})
+ it('never sends the remaining members into the cooldown restored by a member denied again', async () => {
+ // Drives the recovery directly: through apiGet the second member would be
+ // refused locally by the restored cooldown and send nothing either way.
+ const familyKey = 'family-under-test'
+ const denial = Object.assign(new Error('denied again'), { status: 429 })
+ const firstMember = vi.fn(() => Promise.reject(denial))
+ const secondMember = vi.fn(() => Promise.resolve(undefined))
+ const interest = new AbortController()
+ const deadlineMs = Date.now() + 1_000
+ expect(
+ reserveReadRequestRecovery(familyKey, deadlineMs, 'first', firstMember, [interest.signal])
+ ).toBe(true)
+ expect(
+ reserveReadRequestRecovery(familyKey, deadlineMs, 'second', secondMember, [interest.signal])
+ ).toBe(false)
+
+ await vi.advanceTimersByTimeAsync(1_000)
+ // Witness: the recovery ran and reread the first member.
+ expect(firstMember).toHaveBeenCalledOnce()
+ expect(secondMember).not.toHaveBeenCalled()
+ // The denied attempt released its reservation for the next denial.
+ expect(
+ reserveReadRequestRecovery(familyKey, Date.now() + 1_000, 'first', firstMember, [
+ interest.signal,
+ ])
+ ).toBe(true)
+ })
+
+ it('gives a waiter its own recovered result when a later member is denied again', async () => {
+ const codexUrl = '/api/v1/admin/codex-subscription/connections'
+ const grokUrl = '/api/v1/admin/grok-subscription/connections'
+ const connectionOptions = { metadataRead: 'subscription-connections' } as const
+ const pending: Array<(response: Response) => void> = []
+ fetchMock.mockImplementation(
+ () =>
+ new Promise<Response>(resolve => {
+ pending.push(resolve)
+ })
+ )
+ const codexConsumer = new AbortController()
+ const grokConsumer = new AbortController()
+ const codex = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })
+ const grok = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })
+ await vi.advanceTimersByTimeAsync(0)
+ pending.splice(0).forEach(resolve => resolve(throttled(12)))
+ await expect(codex).rejects.toMatchObject({ status: 429 })
+ await expect(grok).rejects.toMatchObject({ status: 429 })
+
+ await vi.advanceTimersByTimeAsync(12_000)
+ expect(pending).toHaveLength(1)
+ const codexWaiter = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })
+ const grokWaiter = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })
+ const codexOutcome = codexWaiter.then(
+ value => ({ value }),
+ reason => ({ reason })
+ )
+ const grokOutcome = grokWaiter.then(
+ value => ({ value }),
+ reason => ({ reason })
+ )
+ pending.shift()?.(success({ from: 'codex' }))
+ await vi.advanceTimersByTimeAsync(0)
+ expect(pending).toHaveLength(1)
+ pending.shift()?.(throttled(12))
+
+ // Codex was reread before Grok's denial and receives its own result; Grok's
+ // waiter receives the denial that ended the recovery.
+ expect(await codexOutcome).toEqual({ value: { from: 'codex' } })
+ expect(await grokOutcome).toMatchObject({ reason: { status: 429 } })
+ expect(fetchMock).toHaveBeenCalledTimes(4)
+ })
+
+ it('reads for a waiter without a signal when every registered interest cancels the recovery', async () => {
+ const pending: Array<(response: Response) => void> = []
+ fetchMock.mockImplementation(
+ () =>
+ new Promise<Response>(resolve => {
+ pending.push(resolve)
+ })
+ )
+ const mounted = new AbortController()
+ const first = apiGet(
+ '/api/v1/admin/metadata',
+ {},
+ { ...metadataOptions, signal: mounted.signal }
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ pending.shift()?.(throttled(2))
+ await expect(first).rejects.toMatchObject({ status: 429 })
+
+ await vi.advanceTimersByTimeAsync(2_000)
+ expect(pending).toHaveLength(1)
+ // A consumer such as the feature probes has no signal and cannot be
+ // registered as an interest.
+ const unsignalled = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
+ const outcome = unsignalled.then(
+ value => ({ value }),
+ reason => ({ reason })
+ )
+ mounted.abort()
+ await vi.advanceTimersByTimeAsync(0)
+ // The cancelled recovery leaves it a read of its own.
+ expect(pending).toHaveLength(2)
+ pending[1](success({ value: 'own read' }))
+ expect(await outcome).toEqual({ value: { value: 'own read' } })
+ expect(fetchMock).toHaveBeenCalledTimes(3)
+ })
+
+ it('keeps a late recovery scheduled when a URL outside it succeeds', async () => {
+ const codexUrl = '/api/v1/admin/codex-subscription/connections'
+ const grokUrl = '/api/v1/admin/grok-subscription/connections'
+ const connectionOptions = { metadataRead: 'subscription-connections' } as const
+ const pending = new Map<string, (response: Response) => void>()
+ fetchMock.mockImplementation(
+ (input: RequestInfo | URL) =>
+ new Promise<Response>(resolve => {
+ pending.set(String(input).includes('grok') ? 'grok' : 'codex', resolve)
+ })
+ )
+ const codex = apiGet(codexUrl, {}, connectionOptions)
+ const grok = apiGet(grokUrl, {}, connectionOptions)
+ await vi.advanceTimersByTimeAsync(0)
+ pending.get('codex')?.(throttled(12))
+ await expect(codex).rejects.toMatchObject({ status: 429 })
+
+ // A background tab runs the recovery timer late: the clock passes the
+ // deadline before the timer fires, and the slow Grok read succeeds first.
+ vi.setSystemTime(Date.now() + 13_000)
+ fetchMock.mockImplementation(() => Promise.resolve(success({ from: 'codex' })))
+ pending.get('grok')?.(success({ from: 'grok' }))
+ await expect(grok).resolves.toEqual({ from: 'grok' })
+ // Fake timers keep their own clock, so the delayed timer fires only now.
+ await vi.advanceTimersByTimeAsync(12_000)
+
+ // The Codex member is still reread.
+ expect(fetchMock).toHaveBeenCalledTimes(3)
+ expect(String(fetchMock.mock.calls[2][0])).toMatch(/codex-subscription\/connections$/)
+ await expect(apiGet(codexUrl, {}, connectionOptions)).resolves.toEqual({ from: 'codex' })
+ expect(fetchMock).toHaveBeenCalledTimes(3)
+ })
+
+ it('moves the retry time of every earlier denial when a later one extends the cooldown', () => {
+ const familyKey = 'family-under-test'
+ const startedAt = Date.now()
+ const earlier = Object.assign(new Error('earlier'), { status: 429 })
+ const later = Object.assign(new Error('later'), { status: 429 })
+ expect(setReadRequestCooldown(familyKey, earlier, 12, startedAt)).toBe(12_000)
+ expect(setReadRequestCooldown(familyKey, later, 12, startedAt + 500)).toBe(12_000)
+ expect(later.retryAtMs).toBe(startedAt + 12_500)
+ expect(earlier.retryAtMs).toBe(startedAt + 12_500)
+
+ // A denial after the cooldown expired starts a new window and leaves the
+ // errors of the expired one alone.
+ const next = Object.assign(new Error('next'), { status: 429 })
+ setReadRequestCooldown(familyKey, next, 12, startedAt + 20_000)
+ expect(next.retryAtMs).toBe(startedAt + 32_000)
+ expect(earlier.retryAtMs).toBe(startedAt + 12_500)
+ })
+
+ it('schedules a retry at the extended time instead of the time first issued', async () => {
+ const familyKey = 'family-under-test'
+ const startedAt = Date.now()
+ const earlier = Object.assign(new Error('earlier'), { status: 429 })
+ setReadRequestCooldown(familyKey, earlier, 12, startedAt)
+ const retry = vi.fn()
+ scheduleReadRequestRetry(earlier, retry)
+ await vi.advanceTimersByTimeAsync(500)
+ setReadRequestCooldown(
+ familyKey,
+ Object.assign(new Error('later'), { status: 429 }),
+ 12,
+ startedAt + 500
+ )
+ await vi.advanceTimersByTimeAsync(11_500)
+ expect(retry).not.toHaveBeenCalled()
+ // Witness: the retry runs once the extended time is reached.
+ await vi.advanceTimersByTimeAsync(500)
+ expect(retry).toHaveBeenCalledOnce()
+ })
+
+ it('changes the session identity only when the confirmed principal changes', () => {
+ const start = getReadRequestSessionIdentity()
+ setReadRequestCacheEntry('entry', { value: 1 }, 30_000)
+ invalidateReadRequestCache()
+ invalidateReadRequestCacheEntry('entry')
+ setControlUIReadPrincipal('admin-one', 'admin')
+ expect(getReadRequestSessionIdentity()).toBe(start)
+ // Witness: a different principal does change it.
+ setControlUIReadPrincipal('admin-two', 'admin')
+ expect(getReadRequestSessionIdentity()).toBe(start + 1)
+ })
+
it('makes only one recovery when throttle timing is unavailable', async () => {
const noTiming = () =>
new Response(JSON.stringify({ error: 'Too Many Requests' }), { status: 429 })
diff --git a/control-ui/lib/api.ts b/control-ui/lib/api.ts
index ddbeda0e2..6747337f8 100644
--- a/control-ui/lib/api.ts
+++ b/control-ui/lib/api.ts
@@ -458,7 +458,23 @@ async function apiRead(
const recovery = getReadRequestRecovery(familyKey)
if (recovery) {
joinReadRequestRecovery(familyKey, options.signal)
- await waitForRead(recovery, options.signal)
+ try {
+ await waitForRead(recovery, options.signal)
+ } catch (error) {
+ assertCurrent()
+ // The recovery rejects with its first failing member. Members reread
+ // before that failure are cached; their waiters get their own result,
+ // not a sibling's denial.
+ const own = metadataKey ? getReadRequestCacheEntry(metadataKey) : undefined
+ if (own !== undefined) return own
+ // Every registered interest left and the recovery was cancelled. A
+ // waiter without a signal cannot be registered, yet it still wants
+ // the result: read for itself now that the deadline has passed.
+ if (error instanceof DOMException && error.name === 'AbortError') {
+ return apiRead(path, query, options)
+ }
+ throw error
+ }
assertCurrent()
return apiRead(path, query, options)
}
@@ -550,10 +566,13 @@ async function apiRead(
error.retryAtMs = Date.now() + seconds * 1000
}
if (familyKey && res.status === 429) {
- const delayMs = setReadRequestCooldown(familyKey, error, seconds)
+ // One clock reading, so the recovery deadline equals the cooldown's
+ // retry time and a consumer released by one is never refused by the other.
+ const deniedAtMs = Date.now()
+ const delayMs = setReadRequestCooldown(familyKey, error, seconds, deniedAtMs)
reserveReadRequestRecovery(
familyKey,
- Date.now() + delayMs,
+ deniedAtMs + delayMs,
metadataKey ?? url,
signal =>
sameReadContext(requestEpoch, requestPrincipalKey)
@@ -567,7 +586,9 @@ async function apiRead(
const parsed = await parseJsonResponse(res)
assertNetworkCurrent()
if (metadataKey) setReadRequestCacheEntry(metadataKey, parsed, METADATA_READ_TTL_MS)
- if (familyKey && !getReadRequestCooldown(familyKey)) completeReadRequestRecovery(familyKey)
+ if (familyKey && metadataKey && !getReadRequestCooldown(familyKey)) {
+ completeReadRequestRecovery(familyKey, metadataKey)
+ }
return parsed
} catch (error) {
assertNetworkCurrent()
diff --git a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx
index 9f5df0421..4be825969 100644
--- a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx
+++ b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx
@@ -1,6 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { act, cleanup, renderHook, waitFor } from '@testing-library/react'
-import { clearAdminAuthToken, setControlUIReadPrincipal } from '../../api'
+import { apiGet, clearAdminAuthToken, setControlUIReadPrincipal } from '../../api'
import { __resetReadRequestCacheForTests } from '../../readRequestCache'
import { SUBSCRIPTION_CAPABILITIES_API_PATH } from '../../subscriptionCapabilities'
import { useSubscriptionCapabilities } from '../useSubscriptionCapabilities'
@@ -61,6 +61,121 @@ describe('useSubscriptionCapabilities with the real read cache', () => {
expect(capabilityReads(fetchMock)).toBe(2)
})
+ describe('automatic throttle recovery', () => {
+ function throttledResponse(retryAfterSeconds: number): Response {
+ return new Response(JSON.stringify({ error: 'Too Many Requests', message: 'limited' }), {
+ status: 429,
+ statusText: '',
+ headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) },
+ })
+ }
+
+ function okResponse(body: unknown): Response {
+ return new Response(JSON.stringify(body), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ })
+ }
+
+ const flush = (ms = 0) =>
+ act(async () => {
+ await vi.advanceTimersByTimeAsync(ms)
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('waits for a sibling denial that extends the family deadline and then shows the recovered result', async () => {
+ vi.useFakeTimers()
+ vi.setSystemTime(new Date('2026-01-01T00:00:48.000Z'))
+ const pending = new Map<string, (response: Response) => void>()
+ fetchMock.mockImplementation(
+ (input: RequestInfo | URL) =>
+ new Promise<Response>(resolve => {
+ pending.set(
+ String(input).includes('capabilities') ? 'capabilities' : 'inventory',
+ resolve
+ )
+ })
+ )
+ const inventory = new AbortController()
+ const sibling = apiGet(
+ '/api/v1/admin/codex-subscription/connections',
+ {},
+ { metadataRead: 'subscription-connections', signal: inventory.signal }
+ ).catch(reason => reason)
+ const consumer = renderHook(() => useSubscriptionCapabilities())
+ await flush()
+ pending.get('capabilities')?.(throttledResponse(12))
+ await flush()
+ expect(consumer.result.current.error?.status).toBe(429)
+ // Half a second later the in-flight sibling is denied too, which moves
+ // the family deadline from :60.0 to :60.5.
+ await flush(500)
+ pending.get('inventory')?.(throttledResponse(12))
+ await flush()
+ expect(await sibling).toMatchObject({ status: 429 })
+
+ fetchMock.mockImplementation((input: RequestInfo | URL) =>
+ Promise.resolve(
+ String(input).includes('capabilities')
+ ? okResponse(capabilitiesBody)
+ : okResponse({ items: [] })
+ )
+ )
+ await flush(20_000)
+ // The recovery read capabilities once and the consumer shows that result
+ // without a manual Retry.
+ expect(capabilityReads(fetchMock)).toBe(2)
+ expect(consumer.result.current.capabilities).toEqual(capabilitiesBody)
+ expect(consumer.result.current.error).toBeNull()
+ })
+
+ it('gives a new session its own automatic attempt after the previous one spent it', async () => {
+ vi.useFakeTimers()
+ fetchMock.mockImplementation(() => Promise.resolve(throttledResponse(2)))
+ const consumer = renderHook(() => useSubscriptionCapabilities())
+ await flush()
+ expect(consumer.result.current.error?.status).toBe(429)
+ // The automatic attempt is denied again; none is left for this session.
+ await flush(2_000)
+ await flush(10_000)
+ const firstSessionReads = capabilityReads(fetchMock)
+ expect(consumer.result.current.error?.status).toBe(429)
+
+ act(() => setControlUIReadPrincipal('admin-two', 'admin'))
+ await flush()
+ expect(capabilityReads(fetchMock)).toBe(firstSessionReads + 1)
+ expect(consumer.result.current.error?.status).toBe(429)
+ fetchMock.mockImplementation(() => Promise.resolve(okResponse(capabilitiesBody)))
+ await flush(2_000)
+ // Witness: the new session's automatic attempt ran and recovered.
+ expect(consumer.result.current.capabilities).toEqual(capabilitiesBody)
+ })
+ })
+
+ it('reads nothing before the first principal is confirmed and loads once it is', async () => {
+ __resetReadRequestCacheForTests()
+ fetchMock.mockResolvedValue(
+ new Response(JSON.stringify(capabilitiesBody), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ })
+ )
+ const consumer = renderHook(() => useSubscriptionCapabilities())
+ await act(async () => {
+ await Promise.resolve()
+ })
+ expect(consumer.result.current.loading).toBe(true)
+ expect(capabilityReads(fetchMock)).toBe(0)
+
+ act(() => setControlUIReadPrincipal('admin-one', 'admin'))
+ // Witness: the confirmed session loads, once.
+ await waitFor(() => expect(consumer.result.current.capabilities).toEqual(capabilitiesBody))
+ expect(capabilityReads(fetchMock)).toBe(1)
+ })
+
describe('when the confirmed principal changes while the consumer stays mounted', () => {
const sessionOneBody = {
providers: {
diff --git a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx
index a248d973e..20805cc98 100644
--- a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx
+++ b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx
@@ -1,7 +1,16 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { act, cleanup, renderHook, waitFor } from '@testing-library/react'
+import { __resetReadRequestCacheForTests, setReadRequestPrincipal } from '../../readRequestCache'
import { useSubscriptionCapabilities } from '../useSubscriptionCapabilities'
+// Capability consumers load only for a principal confirmed by /me (AuthContext).
+beforeEach(() => {
+ setReadRequestPrincipal('admin-one', 'admin')
+})
+afterEach(() => {
+ __resetReadRequestCacheForTests()
+})
+
const load = vi.hoisted(() => vi.fn())
vi.mock('../../subscriptionCapabilities', () => ({
diff --git a/control-ui/lib/hooks/useSubscriptionCapabilities.ts b/control-ui/lib/hooks/useSubscriptionCapabilities.ts
index f04de85e2..16af5be47 100644
--- a/control-ui/lib/hooks/useSubscriptionCapabilities.ts
+++ b/control-ui/lib/hooks/useSubscriptionCapabilities.ts
@@ -1,14 +1,12 @@
'use client'
import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react'
-import {
- METADATA_READ_MAX_RECOVERY_ATTEMPTS,
- METADATA_READ_UNTIMED_COOLDOWN_MS,
-} from '@constants/readRequests'
+import { METADATA_READ_MAX_RECOVERY_ATTEMPTS } from '@constants/readRequests'
import type { ApiRequestError } from '../api.types'
import {
getReadRequestPrincipal,
getReadRequestSessionIdentity,
+ scheduleReadRequestRetry,
subscribeReadRequestSessionIdentity,
} from '../readRequestCache'
import {
@@ -65,17 +63,16 @@ export function useSubscriptionCapabilities(
}, [])
// The confirmed principal. When it changes, the previous session's result,
- // error and recovery budget are dropped and the new session loads its own;
- // with no confirmed principal the hook waits, loading, for /me to confirm one.
+ // error and recovery budget are dropped and the new session loads its own.
+ // With no confirmed principal (before the first /me, or after a session
+ // ended) the hook waits, loading, for /me to confirm one: an unconfirmed
+ // read could neither be cached nor share the family's quota coordination.
const sessionIdentity = useSyncExternalStore(
subscribeReadRequestSessionIdentity,
getReadRequestSessionIdentity,
() => 0
)
const sessionIdentityRef = useRef(sessionIdentity)
- // Set when the session this consumer loaded for ended without a successor;
- // Retry cannot load for an unconfirmed session, only /me can end the wait.
- const sessionEndedRef = useRef(false)
useEffect(() => {
if (sessionIdentityRef.current !== sessionIdentity) {
@@ -83,7 +80,6 @@ export function useSubscriptionCapabilities(
setCapabilities(null)
setError(null)
recoveryAttemptsRef.current = 0
- sessionEndedRef.current = getReadRequestPrincipal() === null
}
if (!enabled) {
requestRef.current += 1
@@ -91,7 +87,8 @@ export function useSubscriptionCapabilities(
setLoading(false)
return
}
- if (sessionEndedRef.current) {
+ // Retry cannot load for an unconfirmed session; only /me can end the wait.
+ if (getReadRequestPrincipal() === null) {
requestRef.current += 1
controllerRef.current?.abort()
setLoading(true)
@@ -111,13 +108,10 @@ export function useSubscriptionCapabilities(
recoveryAttemptsRef.current >= METADATA_READ_MAX_RECOVERY_ATTEMPTS
)
return
- const retryAtMs = error.retryAtMs ?? Date.now() + METADATA_READ_UNTIMED_COOLDOWN_MS
- const delay = Math.max(0, retryAtMs - Date.now())
- const timer = setTimeout(() => {
+ return scheduleReadRequestRetry(error, () => {
recoveryAttemptsRef.current += 1
void load()
- }, delay)
- return () => clearTimeout(timer)
+ })
}, [enabled, error, load])
const retry = useCallback(() => {
diff --git a/control-ui/lib/readRequestCache.ts b/control-ui/lib/readRequestCache.ts
index 7eb253e2a..6f1284605 100644
--- a/control-ui/lib/readRequestCache.ts
+++ b/control-ui/lib/readRequestCache.ts
@@ -13,6 +13,10 @@ type CacheEntry = {
type CooldownEntry = {
error: ApiRequestError
retryAtMs: number
+ // Every denial issued while this cooldown was active. A later denial can
+ // extend the window; each consumer still holding an earlier error must see
+ // the extended retry time, or its own timer fires into the cooldown.
+ issued: Set<ApiRequestError>
}
type RecoveryEntry = {
@@ -208,14 +212,18 @@ export function setReadRequestCooldown(
if (!isBrowser()) return 0
const delayMs =
retryAfterSeconds !== undefined ? retryAfterSeconds * 1000 : METADATA_READ_UNTIMED_COOLDOWN_MS
- const retryAtMs = Math.max(nowMs + delayMs, cooldowns.get(key)?.retryAtMs ?? 0)
- error.retryAtMs = retryAtMs
+ const active = cooldowns.get(key)
+ const current = active && active.retryAtMs > nowMs ? active : undefined
+ const retryAtMs = Math.max(nowMs + delayMs, current?.retryAtMs ?? 0)
+ const issued = current?.issued ?? new Set<ApiRequestError>()
+ issued.add(error)
+ for (const denied of issued) denied.retryAtMs = retryAtMs
while (cooldowns.size >= METADATA_READ_CACHE_MAX_ENTRIES && !cooldowns.has(key)) {
const oldest = cooldowns.keys().next().value
if (oldest === undefined) break
cooldowns.delete(oldest)
}
- cooldowns.set(key, { error, retryAtMs })
+ cooldowns.set(key, { error, retryAtMs, issued })
return retryAtMs - nowMs
}
@@ -363,16 +371,45 @@ async function runRecovery(key: string, entry: RecoveryEntry): Promise<void> {
entry.resolve()
}
-export function completeReadRequestRecovery(key: string): void {
+/**
+ * A member URL read successfully outside the recovery (its timer can fire late
+ * in a background tab) no longer needs its reread. The recovery settles once
+ * no member is left; a success for a URL that is not a member changes nothing.
+ */
+export function completeReadRequestRecovery(key: string, memberKey: string): void {
const recovery = recoveries.get(key)
// A running recovery settles itself once every member has been reread.
if (!recovery || recovery.state === 'running') return
+ if (!recovery.members.delete(memberKey) || recovery.members.size > 0) return
if (recovery.timer) clearTimeout(recovery.timer)
recovery.removeListeners()
recovery.resolve()
recoveries.delete(key)
}
+/**
+ * Run `retry` once a denial's retry time has passed. A later denial in the same
+ * family can extend that time on the error itself (setReadRequestCooldown), so
+ * the timer re-arms until the extended time is reached instead of firing into
+ * the cooldown. Returns the cancel function.
+ */
+export function scheduleReadRequestRetry(error: ApiRequestError, retry: () => void): () => void {
+ const untimedRetryAtMs = Date.now() + METADATA_READ_UNTIMED_COOLDOWN_MS
+ let timer: ReturnType<typeof setTimeout>
+ const arm = () => {
+ const delayMs = (error.retryAtMs ?? untimedRetryAtMs) - Date.now()
+ timer = setTimeout(
+ () => {
+ if (Date.now() < (error.retryAtMs ?? untimedRetryAtMs)) arm()
+ else retry()
+ },
+ Math.min(2_147_483_647, Math.max(0, delayMs))
+ )
+ }
+ arm()
+ return () => clearTimeout(timer)
+}
+
export function __resetReadRequestCacheForTests(): void {
principal = null
if (invalidationChannel) {
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index a9403fe83..dcb58c4f8 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.716",
+ "version": "0.1.717",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.716",
+ "version": "0.1.717",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 7a1d97d58..e151ab6e6 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.716",
+ "version": "0.1.717",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.md
index 9bf13301a..48c5d14ed 100644
--- a/docs/architecture/administrative-rate-limits.md
+++ b/docs/architecture/administrative-rate-limits.md
@@ -153,16 +153,27 @@ subscription inventory; editor opening loads it when needed.
One shared recovery runs per read family after a denial deadline. It rereads,
once each and one at a time, every URL the server denied while it was
scheduled (for example the Codex and Grok inventories loaded together), and
-stops at the first reread that is denied again. A denied recovery schedules no
+stops at the first reread that is denied again. A consumer waiting on it
+receives its own URL's result when that URL was reread before the denial, and
+the denial otherwise. A member read successfully outside the recovery (its
+timer can fire late in a background tab) drops out of it; a success for a URL
+that is not a member leaves it scheduled. A denied recovery schedules no
further attempt, but it releases its reservation, so the next denial in that
-read family schedules its own. A consumer refused by the cooldown, or waiting
-on the recovery after its deadline, joins it as an interest without sending a
-request or changing its deadline, members or budget, so the recovery is
-cancelled only when every
-registered consumer has left; a consumer without an abort signal is not
-registered. Capability consumers follow the confirmed principal: when it
-changes they drop the previous session's result and error and load for the
-new session, and with no confirmed principal they wait for one. Forced refresh respects the same pause. Authentication changes, 401 and
+read family schedules its own. A later denial that extends the family deadline
+also moves the retry time of the denials already shown, so automatic retries
+wait for the extended deadline instead of firing into the cooldown.
+
+A consumer refused by the cooldown, or waiting on the recovery after its
+deadline, joins it as an interest without sending a request or changing its
+deadline, members or budget, so the recovery is cancelled only when every
+registered consumer has left. A consumer without an abort signal is not
+registered; when the recovery it waits on is cancelled that way, it reads for
+itself. Capability consumers and the Agent model editor retry automatically
+once at the deadline, which waits for the recovery and shows its cached result.
+Capability consumers follow the confirmed principal: when it changes they drop
+the previous session's result, error and automatic attempt and load for the
+new session, and with no confirmed principal (before the first /me, or after a
+session ended) they wait for one. Forced refresh respects the same pause. Authentication changes, 401 and
cross-tab invalidation fence old awaits and clear identity-bound state.
Aborting one subscriber does not cancel another subscriber’s shared request.
SSR does not retain cached/deduplicated results across requests.