Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-ui): keep member results and automatic recovery across family denials
Alfredo Lopez25247386e87f
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex aa6d7ce06..6094a5ab6 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.718",+ "version": "0.1.719","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.718",+ "version": "0.1.719","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex ad822f2bb..c12af00a3 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.718",+ "version": "0.1.719","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.ts b/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.tsnew file mode 100644index 000000000..f83e459df--- /dev/null+++ b/control-api/test/routes.mcpSecretsDeleteEdge.wiring.test.ts@@ -0,0 +1,75 @@+import { describe, expect, it, vi } from 'vitest'+import express from 'express'+import request from 'supertest'++vi.mock('../src/services/rateLimiterService.js', () => ({+ checkAndIncrement: vi.fn(async (_key: string, maxPerMinute: number) => ({+ allowed: true,+ remaining: maxPerMinute - 1,+ resetMs: Date.now() + 60_000,+ windowStartMs: Date.now(),+ count: 1,+ backendAvailable: true,+ })),+}))++function createGateway() {+ return {+ deleteSecret: vi.fn(async (name: string, namespace?: string) => ({+ name,+ namespace: namespace || 'mcp-server',+ deleted: true as const,+ })),+ getSecret: vi.fn(async (name: string, namespace?: string) => ({+ metadata: { name, namespace, uid: `uid-${name}`, resourceVersion: '1', labels: {} },+ data: {},+ })),+ listResource: vi.fn(async () => [] as unknown[]),+ }+}++function createRollbackPermitStore() {+ return {+ issue: vi.fn(async () => undefined),+ claim: vi.fn(async () => null),+ release: vi.fn(async () => undefined),+ finalize: vi.fn(async () => undefined),+ }+}++describe('DELETE /admin/mcp-secrets/:name edge wiring', () => {+ it('mounts the calendar-minute edge limiter in front of the delete handler', async () => {+ // The route's limiter is built when the module loads, so lower its limit+ // before importing the router.+ vi.resetModules()+ const { config } = await import('../src/config.js')+ config.adminConnectorDeleteEdgePerMin = 2+ const { checkAndIncrement } = await import('../src/services/rateLimiterService.js')+ const { createAdminSecretsRouter } = await import('../src/routes/admin/secrets.js')+ const gateway = createGateway()+ const app = express()+ app.use(express.json())+ app.use((req, _res, next) => {+ ;(req as express.Request & { adminAuth?: { sub: string; jti: string } }).adminAuth = {+ sub: 'admin-wiring',+ jti: 'admin-wiring-jti',+ }+ next()+ })+ app.use(createAdminSecretsRouter(gateway as never, createRollbackPermitStore() as never))++ const send = () =>+ request(app)+ .delete('/admin/mcp-secrets/wiring-creds')+ .send({ uid: 'uid-wiring-creds', resourceVersion: '1' })+ await send().expect(200)+ await send().expect(200)+ const denied = await send().expect(429)++ // Witness: the two admitted deletes reached the ledger and the handler; the+ // third was refused by the edge before either.+ expect(gateway.deleteSecret).toHaveBeenCalledTimes(2)+ expect(vi.mocked(checkAndIncrement)).toHaveBeenCalledTimes(2)+ expect(denied.headers['ratelimit-policy']).toBe('2;w=60')+ })+})diff --git a/control-ui/app/hosts/[name]/page.tsx b/control-ui/app/hosts/[name]/page.tsxindex 0c25f88a6..4f67825c5 100644--- a/control-ui/app/hosts/[name]/page.tsx+++ b/control-ui/app/hosts/[name]/page.tsx@@ -36,6 +36,7 @@ import {updateContext,} from '../../../lib/api'import type { ContextResource, ContextSpec, HostSecretResource } from '../../../lib/api'+import type { ApiRequestError } from '../../../lib/api.types'import {CODEX_UNASSIGNED_CONNECTION_KEY,type CodexSubscriptionConnectionView,@@ -78,6 +79,7 @@ import {validateLlmPolicy,} from '../../../lib/llm'import { credentialSelectValue, parseCredentialSelect } from '../../../lib/llmCredentialSelect'+import { scheduleReadRequestRetry } from '../../../lib/readRequestCache'import type { HostTab } from './types'const TAB_LABELS: Record<HostTab, string> = {@@ -269,6 +271,9 @@ export default function HostDetailsPage() {const [subscriptionInventoryError, setSubscriptionInventoryError] = useState('')const [subscriptionInventoryLoading, setSubscriptionInventoryLoading] = useState(false)const [inventoryRetryNonce, setInventoryRetryNonce] = useState(0)+ const [subscriptionInventoryThrottle, setSubscriptionInventoryThrottle] =+ useState<ApiRequestError | null>(null)+ const inventoryAutoRetryUsedRef = useRef(false)const catalogForEditor = useMemo(() => {if (providerDraft === 'codex-subscription') {const others = allowedCatalog.filter(row => row.provider !== 'codex-subscription')@@ -426,6 +431,7 @@ export default function HostDetailsPage() {useEffect(() => {if (!editingModel || !subscriptionCapabilities.capabilities) {setSubscriptionInventoryError('')+ setSubscriptionInventoryThrottle(null)setSubscriptionInventoryLoading(false)return}@@ -447,11 +453,15 @@ export default function HostDetailsPage() {setCodexConnections(codexResult.value)setCodexInventoryLoaded(codexEnabled)}+ if (codexResult.status === 'fulfilled' && grokResult?.status !== 'rejected') {+ inventoryAutoRetryUsedRef.current = false+ }if (grokResult?.status === 'fulfilled') {setGrokConnections(grokResult.value)setGrokInventoryLoaded(true)}setSubscriptionInventoryError('')+ setSubscriptionInventoryThrottle(null)const failure = [codexResult, grokResult].find(result => result?.status === 'rejected' && !isDisabledCapabilityError(result.reason))@@ -460,6 +470,8 @@ export default function HostDetailsPage() {.catch(err => {if (controller.signal.aborted || isDisabledCapabilityError(err)) returnsetSubscriptionInventoryError(subscriptionInventoryMessage(err, 'subscription'))+ const denied = err as ApiRequestError+ setSubscriptionInventoryThrottle(denied?.status === 429 ? denied : null)}).finally(() => {if (!controller.signal.aborted) setSubscriptionInventoryLoading(false)@@ -473,6 +485,19 @@ export default function HostDetailsPage() {subscriptionCapabilities.capabilities,])+ // The family's shared recovery rereads a denied inventory once Retry-After+ // passes. Re-run the load once at that time: it waits for the recovery and+ // shows its cached result without a click or a second read. A denial on that+ // run leaves the visible Retry; a success or a manual Retry restores the+ // automatic attempt for the next denial.+ useEffect(() => {+ if (!subscriptionInventoryThrottle || inventoryAutoRetryUsedRef.current) return+ return scheduleReadRequestRetry(subscriptionInventoryThrottle, () => {+ inventoryAutoRetryUsedRef.current = true+ setInventoryRetryNonce(value => value + 1)+ })+ }, [subscriptionInventoryThrottle])+useEffect(() => {if (!editingModel) returnif (!connectionRefDraft.trim() || connectionRefDraft === CODEX_UNASSIGNED_CONNECTION_KEY) {@@ -1519,7 +1544,10 @@ export default function HostDetailsPage() {<buttontype="button"className="cu-btn cu-btn--ghost cu-btn--sm"- onClick={() => setInventoryRetryNonce(value => value + 1)}+ onClick={() => {+ inventoryAutoRetryUsedRef.current = false+ setInventoryRetryNonce(value => value + 1)+ }}>Retry</button>diff --git a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsxindex f66f9df10..1e1b4fc61 100644--- a/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx+++ b/control-ui/components/__tests__/CodexSubscriptionHub.test.tsx@@ -19,9 +19,18 @@ import {revokeGrokSubscription,startGrokDeviceConnect,} from '@lib/grokSubscription'+import { __resetReadRequestCacheForTests, setReadRequestPrincipal } from '@lib/readRequestCache'import { CodexSubscriptionHub } from '../CodexSubscriptionHub'import { ToastProvider } from '../Toast'+// Capability consumers load only for a principal confirmed by /me (AuthContext).+beforeEach(() => {+ setReadRequestPrincipal('admin-one', 'admin')+})+afterEach(() => {+ __resetReadRequestCacheForTests()+})+const confirmMock = vi.fn()const capabilityProbes = vi.hoisted(() => ({diff --git a/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx b/control-ui/components/__tests__/HostDetailsPage.identity.test.tsxindex ed2293451..e6f86358d 100644--- a/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx+++ b/control-ui/components/__tests__/HostDetailsPage.identity.test.tsx@@ -19,8 +19,20 @@ import {listGrokConnectionModels,listGrokSubscriptionConnections,} from '../../lib/grokSubscription'+import {+ __resetReadRequestCacheForTests,+ setReadRequestPrincipal,+} from '../../lib/readRequestCache'import { ToastProvider } from '../Toast'+// Capability consumers load only for a principal confirmed by /me (AuthContext).+beforeEach(() => {+ setReadRequestPrincipal('admin-one', 'admin')+})+afterEach(() => {+ __resetReadRequestCacheForTests()+})+const replaceMock = vi.fn()const pushMock = vi.fn()let mockParams: { name: string; tab?: string } = { name: 'foo' }diff --git a/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx b/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsxindex 339187c43..dab6c2ea8 100644--- a/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx+++ b/control-ui/components/__tests__/HostDetailsPage.rateLimit.test.tsx@@ -278,7 +278,7 @@ describe('HostDetailsPage optional subscription throttling', () => {expect(api.apiSend).not.toHaveBeenCalled()})- it('answers the visible Retry from the background recovery without spending another read in its window', async () => {+ it('clears the error once the background recovery rereads the inventory, without a click or another read', async () => {vi.useFakeTimers()// The server's shared subscription-read quota: two reads per twelve-second window.const windowMs = 12_000@@ -316,20 +316,17 @@ describe('HostDetailsPage optional subscription throttling', () => {expect(alert).toHaveTextContent('Try again in 12 seconds.')expect(reads.map(read => read.status)).toEqual([200, 200, 429])- // The shared recovery reads the denied inventory once the deadline passes.+ // The shared recovery reads the denied inventory once the deadline passes,+ // and the editor shows that result without a click or a read of its own.await act(async () => {await vi.advanceTimersByTimeAsync(windowMs)})- const afterDeadline = () => reads.filter(read => read.window >= 1)- expect(afterDeadline()).toHaveLength(1)- expect(afterDeadline()[0]?.status).toBe(200)-- fireEvent.click(within(alert).getByRole('button', { name: 'Retry' }))await flush()- // Witness: the visible error cleared through the Retry.+ const afterDeadline = reads.filter(read => read.window >= 1)+ expect(afterDeadline).toHaveLength(1)+ expect(afterDeadline[0]?.status).toBe(200)expect(within(dialog).queryByRole('alert')).toBeNull()- expect(afterDeadline().map(read => read.status)).not.toContain(429)- expect(afterDeadline().length).toBeLessThanOrEqual(2)+ expect(screen.queryByText('Loading subscription options…')).toBeNull()expect(api.apiSend).not.toHaveBeenCalled()})diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.tsindex bc412e28b..47253ae10 100644--- a/control-ui/lib/__tests__/readRequestCache.test.ts+++ b/control-ui/lib/__tests__/readRequestCache.test.ts@@ -13,8 +13,13 @@ import {__resetReadRequestCacheForTests,getReadRequestCacheEntry,getReadRequestPrincipal,+ getReadRequestSessionIdentity,invalidateReadRequestCache,+ invalidateReadRequestCacheEntry,+ reserveReadRequestRecovery,+ scheduleReadRequestRetry,setReadRequestCacheEntry,+ setReadRequestCooldown,} from '../readRequestCache'const metadataOptions = { metadataRead: 'subscription-capabilities' } as const@@ -432,6 +437,198 @@ describe('bounded metadata read reuse', () => {expect(fetchMock).toHaveBeenCalledOnce()})+ it('never sends the remaining members into the cooldown restored by a member denied again', async () => {+ // Drives the recovery directly: through apiGet the second member would be+ // refused locally by the restored cooldown and send nothing either way.+ const familyKey = 'family-under-test'+ const denial = Object.assign(new Error('denied again'), { status: 429 })+ const firstMember = vi.fn(() => Promise.reject(denial))+ const secondMember = vi.fn(() => Promise.resolve(undefined))+ const interest = new AbortController()+ const deadlineMs = Date.now() + 1_000+ expect(+ reserveReadRequestRecovery(familyKey, deadlineMs, 'first', firstMember, [interest.signal])+ ).toBe(true)+ expect(+ reserveReadRequestRecovery(familyKey, deadlineMs, 'second', secondMember, [interest.signal])+ ).toBe(false)++ await vi.advanceTimersByTimeAsync(1_000)+ // Witness: the recovery ran and reread the first member.+ expect(firstMember).toHaveBeenCalledOnce()+ expect(secondMember).not.toHaveBeenCalled()+ // The denied attempt released its reservation for the next denial.+ expect(+ reserveReadRequestRecovery(familyKey, Date.now() + 1_000, 'first', firstMember, [+ interest.signal,+ ])+ ).toBe(true)+ })++ it('gives a waiter its own recovered result when a later member is denied again', async () => {+ const codexUrl = '/api/v1/admin/codex-subscription/connections'+ const grokUrl = '/api/v1/admin/grok-subscription/connections'+ const connectionOptions = { metadataRead: 'subscription-connections' } as const+ const pending: Array<(response: Response) => void> = []+ fetchMock.mockImplementation(+ () =>+ new Promise<Response>(resolve => {+ pending.push(resolve)+ })+ )+ const codexConsumer = new AbortController()+ const grokConsumer = new AbortController()+ const codex = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })+ const grok = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })+ await vi.advanceTimersByTimeAsync(0)+ pending.splice(0).forEach(resolve => resolve(throttled(12)))+ await expect(codex).rejects.toMatchObject({ status: 429 })+ await expect(grok).rejects.toMatchObject({ status: 429 })++ await vi.advanceTimersByTimeAsync(12_000)+ expect(pending).toHaveLength(1)+ const codexWaiter = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })+ const grokWaiter = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })+ const codexOutcome = codexWaiter.then(+ value => ({ value }),+ reason => ({ reason })+ )+ const grokOutcome = grokWaiter.then(+ value => ({ value }),+ reason => ({ reason })+ )+ pending.shift()?.(success({ from: 'codex' }))+ await vi.advanceTimersByTimeAsync(0)+ expect(pending).toHaveLength(1)+ pending.shift()?.(throttled(12))++ // Codex was reread before Grok's denial and receives its own result; Grok's+ // waiter receives the denial that ended the recovery.+ expect(await codexOutcome).toEqual({ value: { from: 'codex' } })+ expect(await grokOutcome).toMatchObject({ reason: { status: 429 } })+ expect(fetchMock).toHaveBeenCalledTimes(4)+ })++ it('reads for a waiter without a signal when every registered interest cancels the recovery', async () => {+ const pending: Array<(response: Response) => void> = []+ fetchMock.mockImplementation(+ () =>+ new Promise<Response>(resolve => {+ pending.push(resolve)+ })+ )+ const mounted = new AbortController()+ const first = apiGet(+ '/api/v1/admin/metadata',+ {},+ { ...metadataOptions, signal: mounted.signal }+ )+ await vi.advanceTimersByTimeAsync(0)+ pending.shift()?.(throttled(2))+ await expect(first).rejects.toMatchObject({ status: 429 })++ await vi.advanceTimersByTimeAsync(2_000)+ expect(pending).toHaveLength(1)+ // A consumer such as the feature probes has no signal and cannot be+ // registered as an interest.+ const unsignalled = apiGet('/api/v1/admin/metadata', {}, metadataOptions)+ const outcome = unsignalled.then(+ value => ({ value }),+ reason => ({ reason })+ )+ mounted.abort()+ await vi.advanceTimersByTimeAsync(0)+ // The cancelled recovery leaves it a read of its own.+ expect(pending).toHaveLength(2)+ pending[1](success({ value: 'own read' }))+ expect(await outcome).toEqual({ value: { value: 'own read' } })+ expect(fetchMock).toHaveBeenCalledTimes(3)+ })++ it('keeps a late recovery scheduled when a URL outside it succeeds', async () => {+ const codexUrl = '/api/v1/admin/codex-subscription/connections'+ const grokUrl = '/api/v1/admin/grok-subscription/connections'+ const connectionOptions = { metadataRead: 'subscription-connections' } as const+ const pending = new Map<string, (response: Response) => void>()+ fetchMock.mockImplementation(+ (input: RequestInfo | URL) =>+ new Promise<Response>(resolve => {+ pending.set(String(input).includes('grok') ? 'grok' : 'codex', resolve)+ })+ )+ const codex = apiGet(codexUrl, {}, connectionOptions)+ const grok = apiGet(grokUrl, {}, connectionOptions)+ await vi.advanceTimersByTimeAsync(0)+ pending.get('codex')?.(throttled(12))+ await expect(codex).rejects.toMatchObject({ status: 429 })++ // A background tab runs the recovery timer late: the clock passes the+ // deadline before the timer fires, and the slow Grok read succeeds first.+ vi.setSystemTime(Date.now() + 13_000)+ fetchMock.mockImplementation(() => Promise.resolve(success({ from: 'codex' })))+ pending.get('grok')?.(success({ from: 'grok' }))+ await expect(grok).resolves.toEqual({ from: 'grok' })+ // Fake timers keep their own clock, so the delayed timer fires only now.+ await vi.advanceTimersByTimeAsync(12_000)++ // The Codex member is still reread.+ expect(fetchMock).toHaveBeenCalledTimes(3)+ expect(String(fetchMock.mock.calls[2][0])).toMatch(/codex-subscription\/connections$/)+ await expect(apiGet(codexUrl, {}, connectionOptions)).resolves.toEqual({ from: 'codex' })+ expect(fetchMock).toHaveBeenCalledTimes(3)+ })++ it('moves the retry time of every earlier denial when a later one extends the cooldown', () => {+ const familyKey = 'family-under-test'+ const startedAt = Date.now()+ const earlier = Object.assign(new Error('earlier'), { status: 429 })+ const later = Object.assign(new Error('later'), { status: 429 })+ expect(setReadRequestCooldown(familyKey, earlier, 12, startedAt)).toBe(12_000)+ expect(setReadRequestCooldown(familyKey, later, 12, startedAt + 500)).toBe(12_000)+ expect(later.retryAtMs).toBe(startedAt + 12_500)+ expect(earlier.retryAtMs).toBe(startedAt + 12_500)++ // A denial after the cooldown expired starts a new window and leaves the+ // errors of the expired one alone.+ const next = Object.assign(new Error('next'), { status: 429 })+ setReadRequestCooldown(familyKey, next, 12, startedAt + 20_000)+ expect(next.retryAtMs).toBe(startedAt + 32_000)+ expect(earlier.retryAtMs).toBe(startedAt + 12_500)+ })++ it('schedules a retry at the extended time instead of the time first issued', async () => {+ const familyKey = 'family-under-test'+ const startedAt = Date.now()+ const earlier = Object.assign(new Error('earlier'), { status: 429 })+ setReadRequestCooldown(familyKey, earlier, 12, startedAt)+ const retry = vi.fn()+ scheduleReadRequestRetry(earlier, retry)+ await vi.advanceTimersByTimeAsync(500)+ setReadRequestCooldown(+ familyKey,+ Object.assign(new Error('later'), { status: 429 }),+ 12,+ startedAt + 500+ )+ await vi.advanceTimersByTimeAsync(11_500)+ expect(retry).not.toHaveBeenCalled()+ // Witness: the retry runs once the extended time is reached.+ await vi.advanceTimersByTimeAsync(500)+ expect(retry).toHaveBeenCalledOnce()+ })++ it('changes the session identity only when the confirmed principal changes', () => {+ const start = getReadRequestSessionIdentity()+ setReadRequestCacheEntry('entry', { value: 1 }, 30_000)+ invalidateReadRequestCache()+ invalidateReadRequestCacheEntry('entry')+ setControlUIReadPrincipal('admin-one', 'admin')+ expect(getReadRequestSessionIdentity()).toBe(start)+ // Witness: a different principal does change it.+ setControlUIReadPrincipal('admin-two', 'admin')+ expect(getReadRequestSessionIdentity()).toBe(start + 1)+ })+it('makes only one recovery when throttle timing is unavailable', async () => {const noTiming = () =>new Response(JSON.stringify({ error: 'Too Many Requests' }), { status: 429 })diff --git a/control-ui/lib/api.ts b/control-ui/lib/api.tsindex ddbeda0e2..6747337f8 100644--- a/control-ui/lib/api.ts+++ b/control-ui/lib/api.ts@@ -458,7 +458,23 @@ async function apiRead(const recovery = getReadRequestRecovery(familyKey)if (recovery) {joinReadRequestRecovery(familyKey, options.signal)- await waitForRead(recovery, options.signal)+ try {+ await waitForRead(recovery, options.signal)+ } catch (error) {+ assertCurrent()+ // The recovery rejects with its first failing member. Members reread+ // before that failure are cached; their waiters get their own result,+ // not a sibling's denial.+ const own = metadataKey ? getReadRequestCacheEntry(metadataKey) : undefined+ if (own !== undefined) return own+ // Every registered interest left and the recovery was cancelled. A+ // waiter without a signal cannot be registered, yet it still wants+ // the result: read for itself now that the deadline has passed.+ if (error instanceof DOMException && error.name === 'AbortError') {+ return apiRead(path, query, options)+ }+ throw error+ }assertCurrent()return apiRead(path, query, options)}@@ -550,10 +566,13 @@ async function apiRead(error.retryAtMs = Date.now() + seconds * 1000}if (familyKey && res.status === 429) {- const delayMs = setReadRequestCooldown(familyKey, error, seconds)+ // One clock reading, so the recovery deadline equals the cooldown's+ // retry time and a consumer released by one is never refused by the other.+ const deniedAtMs = Date.now()+ const delayMs = setReadRequestCooldown(familyKey, error, seconds, deniedAtMs)reserveReadRequestRecovery(familyKey,- Date.now() + delayMs,+ deniedAtMs + delayMs,metadataKey ?? url,signal =>sameReadContext(requestEpoch, requestPrincipalKey)@@ -567,7 +586,9 @@ async function apiRead(const parsed = await parseJsonResponse(res)assertNetworkCurrent()if (metadataKey) setReadRequestCacheEntry(metadataKey, parsed, METADATA_READ_TTL_MS)- if (familyKey && !getReadRequestCooldown(familyKey)) completeReadRequestRecovery(familyKey)+ if (familyKey && metadataKey && !getReadRequestCooldown(familyKey)) {+ completeReadRequestRecovery(familyKey, metadataKey)+ }return parsed} catch (error) {assertNetworkCurrent()diff --git a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsxindex 9f5df0421..4be825969 100644--- a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx+++ b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.cache.test.tsx@@ -1,6 +1,6 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { act, cleanup, renderHook, waitFor } from '@testing-library/react'-import { clearAdminAuthToken, setControlUIReadPrincipal } from '../../api'+import { apiGet, clearAdminAuthToken, setControlUIReadPrincipal } from '../../api'import { __resetReadRequestCacheForTests } from '../../readRequestCache'import { SUBSCRIPTION_CAPABILITIES_API_PATH } from '../../subscriptionCapabilities'import { useSubscriptionCapabilities } from '../useSubscriptionCapabilities'@@ -61,6 +61,121 @@ describe('useSubscriptionCapabilities with the real read cache', () => {expect(capabilityReads(fetchMock)).toBe(2)})+ describe('automatic throttle recovery', () => {+ function throttledResponse(retryAfterSeconds: number): Response {+ return new Response(JSON.stringify({ error: 'Too Many Requests', message: 'limited' }), {+ status: 429,+ statusText: '',+ headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) },+ })+ }++ function okResponse(body: unknown): Response {+ return new Response(JSON.stringify(body), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })+ }++ const flush = (ms = 0) =>+ act(async () => {+ await vi.advanceTimersByTimeAsync(ms)+ })++ afterEach(() => {+ vi.useRealTimers()+ })++ it('waits for a sibling denial that extends the family deadline and then shows the recovered result', async () => {+ vi.useFakeTimers()+ vi.setSystemTime(new Date('2026-01-01T00:00:48.000Z'))+ const pending = new Map<string, (response: Response) => void>()+ fetchMock.mockImplementation(+ (input: RequestInfo | URL) =>+ new Promise<Response>(resolve => {+ pending.set(+ String(input).includes('capabilities') ? 'capabilities' : 'inventory',+ resolve+ )+ })+ )+ const inventory = new AbortController()+ const sibling = apiGet(+ '/api/v1/admin/codex-subscription/connections',+ {},+ { metadataRead: 'subscription-connections', signal: inventory.signal }+ ).catch(reason => reason)+ const consumer = renderHook(() => useSubscriptionCapabilities())+ await flush()+ pending.get('capabilities')?.(throttledResponse(12))+ await flush()+ expect(consumer.result.current.error?.status).toBe(429)+ // Half a second later the in-flight sibling is denied too, which moves+ // the family deadline from :60.0 to :60.5.+ await flush(500)+ pending.get('inventory')?.(throttledResponse(12))+ await flush()+ expect(await sibling).toMatchObject({ status: 429 })++ fetchMock.mockImplementation((input: RequestInfo | URL) =>+ Promise.resolve(+ String(input).includes('capabilities')+ ? okResponse(capabilitiesBody)+ : okResponse({ items: [] })+ )+ )+ await flush(20_000)+ // The recovery read capabilities once and the consumer shows that result+ // without a manual Retry.+ expect(capabilityReads(fetchMock)).toBe(2)+ expect(consumer.result.current.capabilities).toEqual(capabilitiesBody)+ expect(consumer.result.current.error).toBeNull()+ })++ it('gives a new session its own automatic attempt after the previous one spent it', async () => {+ vi.useFakeTimers()+ fetchMock.mockImplementation(() => Promise.resolve(throttledResponse(2)))+ const consumer = renderHook(() => useSubscriptionCapabilities())+ await flush()+ expect(consumer.result.current.error?.status).toBe(429)+ // The automatic attempt is denied again; none is left for this session.+ await flush(2_000)+ await flush(10_000)+ const firstSessionReads = capabilityReads(fetchMock)+ expect(consumer.result.current.error?.status).toBe(429)++ act(() => setControlUIReadPrincipal('admin-two', 'admin'))+ await flush()+ expect(capabilityReads(fetchMock)).toBe(firstSessionReads + 1)+ expect(consumer.result.current.error?.status).toBe(429)+ fetchMock.mockImplementation(() => Promise.resolve(okResponse(capabilitiesBody)))+ await flush(2_000)+ // Witness: the new session's automatic attempt ran and recovered.+ expect(consumer.result.current.capabilities).toEqual(capabilitiesBody)+ })+ })++ it('reads nothing before the first principal is confirmed and loads once it is', async () => {+ __resetReadRequestCacheForTests()+ fetchMock.mockResolvedValue(+ new Response(JSON.stringify(capabilitiesBody), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })+ )+ const consumer = renderHook(() => useSubscriptionCapabilities())+ await act(async () => {+ await Promise.resolve()+ })+ expect(consumer.result.current.loading).toBe(true)+ expect(capabilityReads(fetchMock)).toBe(0)++ act(() => setControlUIReadPrincipal('admin-one', 'admin'))+ // Witness: the confirmed session loads, once.+ await waitFor(() => expect(consumer.result.current.capabilities).toEqual(capabilitiesBody))+ expect(capabilityReads(fetchMock)).toBe(1)+ })+describe('when the confirmed principal changes while the consumer stays mounted', () => {const sessionOneBody = {providers: {diff --git a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsxindex a248d973e..20805cc98 100644--- a/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx+++ b/control-ui/lib/hooks/__tests__/useSubscriptionCapabilities.test.tsx@@ -1,7 +1,16 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { act, cleanup, renderHook, waitFor } from '@testing-library/react'+import { __resetReadRequestCacheForTests, setReadRequestPrincipal } from '../../readRequestCache'import { useSubscriptionCapabilities } from '../useSubscriptionCapabilities'+// Capability consumers load only for a principal confirmed by /me (AuthContext).+beforeEach(() => {+ setReadRequestPrincipal('admin-one', 'admin')+})+afterEach(() => {+ __resetReadRequestCacheForTests()+})+const load = vi.hoisted(() => vi.fn())vi.mock('../../subscriptionCapabilities', () => ({diff --git a/control-ui/lib/hooks/useSubscriptionCapabilities.ts b/control-ui/lib/hooks/useSubscriptionCapabilities.tsindex f04de85e2..16af5be47 100644--- a/control-ui/lib/hooks/useSubscriptionCapabilities.ts+++ b/control-ui/lib/hooks/useSubscriptionCapabilities.ts@@ -1,14 +1,12 @@'use client'import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react'-import {- METADATA_READ_MAX_RECOVERY_ATTEMPTS,- METADATA_READ_UNTIMED_COOLDOWN_MS,-} from '@constants/readRequests'+import { METADATA_READ_MAX_RECOVERY_ATTEMPTS } from '@constants/readRequests'import type { ApiRequestError } from '../api.types'import {getReadRequestPrincipal,getReadRequestSessionIdentity,+ scheduleReadRequestRetry,subscribeReadRequestSessionIdentity,} from '../readRequestCache'import {@@ -65,17 +63,16 @@ export function useSubscriptionCapabilities(}, [])// The confirmed principal. When it changes, the previous session's result,- // error and recovery budget are dropped and the new session loads its own;- // with no confirmed principal the hook waits, loading, for /me to confirm one.+ // error and recovery budget are dropped and the new session loads its own.+ // With no confirmed principal (before the first /me, or after a session+ // ended) the hook waits, loading, for /me to confirm one: an unconfirmed+ // read could neither be cached nor share the family's quota coordination.const sessionIdentity = useSyncExternalStore(subscribeReadRequestSessionIdentity,getReadRequestSessionIdentity,() => 0)const sessionIdentityRef = useRef(sessionIdentity)- // Set when the session this consumer loaded for ended without a successor;- // Retry cannot load for an unconfirmed session, only /me can end the wait.- const sessionEndedRef = useRef(false)useEffect(() => {if (sessionIdentityRef.current !== sessionIdentity) {@@ -83,7 +80,6 @@ export function useSubscriptionCapabilities(setCapabilities(null)setError(null)recoveryAttemptsRef.current = 0- sessionEndedRef.current = getReadRequestPrincipal() === null}if (!enabled) {requestRef.current += 1@@ -91,7 +87,8 @@ export function useSubscriptionCapabilities(setLoading(false)return}- if (sessionEndedRef.current) {+ // Retry cannot load for an unconfirmed session; only /me can end the wait.+ if (getReadRequestPrincipal() === null) {requestRef.current += 1controllerRef.current?.abort()setLoading(true)@@ -111,13 +108,10 @@ export function useSubscriptionCapabilities(recoveryAttemptsRef.current >= METADATA_READ_MAX_RECOVERY_ATTEMPTS)return- const retryAtMs = error.retryAtMs ?? Date.now() + METADATA_READ_UNTIMED_COOLDOWN_MS- const delay = Math.max(0, retryAtMs - Date.now())- const timer = setTimeout(() => {+ return scheduleReadRequestRetry(error, () => {recoveryAttemptsRef.current += 1void load()- }, delay)- return () => clearTimeout(timer)+ })}, [enabled, error, load])const retry = useCallback(() => {diff --git a/control-ui/lib/readRequestCache.ts b/control-ui/lib/readRequestCache.tsindex 7eb253e2a..6f1284605 100644--- a/control-ui/lib/readRequestCache.ts+++ b/control-ui/lib/readRequestCache.ts@@ -13,6 +13,10 @@ type CacheEntry = {type CooldownEntry = {error: ApiRequestErrorretryAtMs: number+ // Every denial issued while this cooldown was active. A later denial can+ // extend the window; each consumer still holding an earlier error must see+ // the extended retry time, or its own timer fires into the cooldown.+ issued: Set<ApiRequestError>}type RecoveryEntry = {@@ -208,14 +212,18 @@ export function setReadRequestCooldown(if (!isBrowser()) return 0const delayMs =retryAfterSeconds !== undefined ? retryAfterSeconds * 1000 : METADATA_READ_UNTIMED_COOLDOWN_MS- const retryAtMs = Math.max(nowMs + delayMs, cooldowns.get(key)?.retryAtMs ?? 0)- error.retryAtMs = retryAtMs+ const active = cooldowns.get(key)+ const current = active && active.retryAtMs > nowMs ? active : undefined+ const retryAtMs = Math.max(nowMs + delayMs, current?.retryAtMs ?? 0)+ const issued = current?.issued ?? new Set<ApiRequestError>()+ issued.add(error)+ for (const denied of issued) denied.retryAtMs = retryAtMswhile (cooldowns.size >= METADATA_READ_CACHE_MAX_ENTRIES && !cooldowns.has(key)) {const oldest = cooldowns.keys().next().valueif (oldest === undefined) breakcooldowns.delete(oldest)}- cooldowns.set(key, { error, retryAtMs })+ cooldowns.set(key, { error, retryAtMs, issued })return retryAtMs - nowMs}@@ -363,16 +371,45 @@ async function runRecovery(key: string, entry: RecoveryEntry): Promise<void> {entry.resolve()}-export function completeReadRequestRecovery(key: string): void {+/**+ * A member URL read successfully outside the recovery (its timer can fire late+ * in a background tab) no longer needs its reread. The recovery settles once+ * no member is left; a success for a URL that is not a member changes nothing.+ */+export function completeReadRequestRecovery(key: string, memberKey: string): void {const recovery = recoveries.get(key)// A running recovery settles itself once every member has been reread.if (!recovery || recovery.state === 'running') return+ if (!recovery.members.delete(memberKey) || recovery.members.size > 0) returnif (recovery.timer) clearTimeout(recovery.timer)recovery.removeListeners()recovery.resolve()recoveries.delete(key)}+/**+ * Run `retry` once a denial's retry time has passed. A later denial in the same+ * family can extend that time on the error itself (setReadRequestCooldown), so+ * the timer re-arms until the extended time is reached instead of firing into+ * the cooldown. Returns the cancel function.+ */+export function scheduleReadRequestRetry(error: ApiRequestError, retry: () => void): () => void {+ const untimedRetryAtMs = Date.now() + METADATA_READ_UNTIMED_COOLDOWN_MS+ let timer: ReturnType<typeof setTimeout>+ const arm = () => {+ const delayMs = (error.retryAtMs ?? untimedRetryAtMs) - Date.now()+ timer = setTimeout(+ () => {+ if (Date.now() < (error.retryAtMs ?? untimedRetryAtMs)) arm()+ else retry()+ },+ Math.min(2_147_483_647, Math.max(0, delayMs))+ )+ }+ arm()+ return () => clearTimeout(timer)+}+export function __resetReadRequestCacheForTests(): void {principal = nullif (invalidationChannel) {diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex a9403fe83..dcb58c4f8 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.716",+ "version": "0.1.717","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.716",+ "version": "0.1.717","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 7a1d97d58..e151ab6e6 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.716",+ "version": "0.1.717","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.mdindex 9bf13301a..48c5d14ed 100644--- a/docs/architecture/administrative-rate-limits.md+++ b/docs/architecture/administrative-rate-limits.md@@ -153,16 +153,27 @@ subscription inventory; editor opening loads it when needed.One shared recovery runs per read family after a denial deadline. It rereads,once each and one at a time, every URL the server denied while it wasscheduled (for example the Codex and Grok inventories loaded together), and-stops at the first reread that is denied again. A denied recovery schedules no+stops at the first reread that is denied again. A consumer waiting on it+receives its own URL's result when that URL was reread before the denial, and+the denial otherwise. A member read successfully outside the recovery (its+timer can fire late in a background tab) drops out of it; a success for a URL+that is not a member leaves it scheduled. A denied recovery schedules nofurther attempt, but it releases its reservation, so the next denial in that-read family schedules its own. A consumer refused by the cooldown, or waiting-on the recovery after its deadline, joins it as an interest without sending a-request or changing its deadline, members or budget, so the recovery is-cancelled only when every-registered consumer has left; a consumer without an abort signal is not-registered. Capability consumers follow the confirmed principal: when it-changes they drop the previous session's result and error and load for the-new session, and with no confirmed principal they wait for one. Forced refresh respects the same pause. Authentication changes, 401 and+read family schedules its own. A later denial that extends the family deadline+also moves the retry time of the denials already shown, so automatic retries+wait for the extended deadline instead of firing into the cooldown.++A consumer refused by the cooldown, or waiting on the recovery after its+deadline, joins it as an interest without sending a request or changing its+deadline, members or budget, so the recovery is cancelled only when every+registered consumer has left. A consumer without an abort signal is not+registered; when the recovery it waits on is cancelled that way, it reads for+itself. Capability consumers and the Agent model editor retry automatically+once at the deadline, which waits for the recovery and shows its cached result.+Capability consumers follow the confirmed principal: when it changes they drop+the previous session's result, error and automatic attempt and load for the+new session, and with no confirmed principal (before the first /me, or after a+session ended) they wait for one. Forced refresh respects the same pause. Authentication changes, 401 andcross-tab invalidation fence old awaits and clear identity-bound state.Aborting one subscriber does not cancel another subscriber’s shared request.SSR does not retain cached/deduplicated results across requests.