Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
apiGet,
apiSend,
clearAdminAuthToken,
loginControlUI,
logoutControlUI,
setControlUIReadPrincipal,
setGlobalAuthErrorHandler,
} from '../api'
import { patchCodexCatalogModel } from '../codexSubscription'
import {
__resetReadRequestCacheForTests,
getReadRequestCacheEntry,
getReadRequestPrincipal,
getReadRequestSessionIdentity,
invalidateReadRequestCache,
invalidateReadRequestCacheEntry,
reserveReadRequestRecovery,
scheduleReadRequestRetry,
setReadRequestCacheEntry,
setReadRequestCooldown,
} from '../readRequestCache'
const metadataOptions = { metadataRead: 'subscription-capabilities' } as const
function success(value: unknown): Response {
return new Response(JSON.stringify(value), {
status: 200,
headers: { 'content-type': 'application/json' },
})
}
function throttled(retryAfterSeconds = 12): Response {
return new Response(
JSON.stringify({
error: 'Too Many Requests',
code: 'rate_limited',
message: `This request limit has been reached. Try again in ${retryAfterSeconds} seconds.`,
retryAfterSeconds,
}),
{
status: 429,
statusText: '',
headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) },
}
)
}
describe('bounded metadata read reuse', () => {
let fetchMock: ReturnType<typeof vi.fn>
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'))
__resetReadRequestCacheForTests()
setControlUIReadPrincipal('admin-one', 'admin')
fetchMock = vi.fn()
vi.stubGlobal('fetch', fetchMock)
})
afterEach(() => {
__resetReadRequestCacheForTests()
vi.useRealTimers()
vi.unstubAllGlobals()
vi.restoreAllMocks()
setGlobalAuthErrorHandler(undefined)
})
it('deduplicates simultaneous metadata reads and expires the result after thirty seconds', async () => {
fetchMock.mockReturnValue(new Promise(resolve => resolve(success({ value: 'first' }))))
const first = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const second = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
await expect(first).resolves.toEqual({ value: 'first' })
await expect(second).resolves.toEqual({ value: 'first' })
expect(fetchMock).toHaveBeenCalledOnce()
fetchMock.mockClear()
fetchMock.mockReturnValue(new Promise(resolve => resolve(success({ value: 'second' }))))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
value: 'first',
})
expect(fetchMock).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(30_001)
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
value: 'second',
})
expect(fetchMock).toHaveBeenCalledOnce()
})
it('forces a metadata refresh past the cache and invalidates after a relevant mutation', async () => {
fetchMock
.mockResolvedValueOnce(success({ revision: 1 }))
.mockResolvedValueOnce(success({ revision: 2 }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 1,
})
await expect(
apiGet('/api/v1/admin/metadata', {}, { ...metadataOptions, refresh: true })
).resolves.toEqual({ revision: 2 })
expect(fetchMock).toHaveBeenCalledTimes(2)
fetchMock.mockResolvedValueOnce(success({ models: [] }))
await patchCodexCatalogModel('team-subscription', 'model-one', true)
fetchMock.mockResolvedValueOnce(success({ revision: 3 }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 3,
})
expect(fetchMock).toHaveBeenCalledTimes(4)
})
it('bounds the metadata cache to one hundred twenty-eight entries', async () => {
for (let index = 0; index <= 128; index += 1) {
fetchMock.mockResolvedValueOnce(success({ index }))
await apiGet(`/api/v1/admin/metadata/${index}`, {}, metadataOptions)
}
expect(fetchMock).toHaveBeenCalledTimes(129)
// The newest and the oldest retained entries are served from the cache, so
// the bound evicts exactly one entry rather than caching nothing.
fetchMock.mockClear()
await expect(apiGet('/api/v1/admin/metadata/128', {}, metadataOptions)).resolves.toEqual({
index: 128,
})
await expect(apiGet('/api/v1/admin/metadata/1', {}, metadataOptions)).resolves.toEqual({
index: 1,
})
expect(fetchMock).not.toHaveBeenCalled()
fetchMock.mockResolvedValueOnce(success({ index: 'fresh-first' }))
await expect(apiGet('/api/v1/admin/metadata/0', {}, metadataOptions)).resolves.toEqual({
index: 'fresh-first',
})
expect(fetchMock).toHaveBeenCalledOnce()
})
it('shares one cooldown and at most one recovery after Retry-After', async () => {
fetchMock.mockImplementationOnce(() => Promise.resolve(throttled(12)))
const firstError = (await apiGet('/api/v1/admin/throttled', {}, metadataOptions).catch(
reason => reason
)) as Error & { retryAfterSeconds?: number }
expect(firstError.message).toBe('This request limit has been reached. Try again in 12 seconds.')
expect(firstError.retryAfterSeconds).toBe(12)
const immediateError = (await apiGet('/api/v1/admin/throttled', {}, metadataOptions).catch(
reason => reason
)) as Error
expect(immediateError).toBe(firstError)
expect(fetchMock).toHaveBeenCalledOnce()
fetchMock.mockImplementationOnce(() => Promise.resolve(throttled(12)))
await vi.advanceTimersByTimeAsync(12_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(24_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('shares the denied subscription-read quota across URLs and forced refreshes', async () => {
fetchMock.mockResolvedValueOnce(throttled(12))
const error = await apiGet('/api/v1/admin/capabilities', {}, metadataOptions).catch(
reason => reason
)
await expect(
apiGet(
'/api/v1/admin/connections',
{},
{
metadataRead: 'subscription-connections',
refresh: true,
}
)
).rejects.toBe(error)
await expect(
apiGet(
'/api/v1/admin/models',
{},
{
metadataRead: 'subscription-model-catalog',
}
)
).rejects.toBe(error)
expect(fetchMock).toHaveBeenCalledOnce()
let resolveRecovery!: (response: Response) => void
fetchMock.mockImplementationOnce(
() =>
new Promise<Response>(resolve => {
resolveRecovery = resolve
})
)
await vi.advanceTimersByTimeAsync(12_000)
const connections = apiGet(
'/api/v1/admin/connections',
{},
{
metadataRead: 'subscription-connections',
}
)
const models = apiGet(
'/api/v1/admin/models',
{},
{
metadataRead: 'subscription-model-catalog',
}
)
await vi.advanceTimersByTimeAsync(0)
expect(fetchMock).toHaveBeenCalledTimes(2)
fetchMock
.mockResolvedValueOnce(success({ connections: [] }))
.mockResolvedValueOnce(success({ models: [] }))
resolveRecovery(success({ providers: {} }))
await expect(connections).resolves.toEqual({ connections: [] })
await expect(models).resolves.toEqual({ models: [] })
expect(fetchMock).toHaveBeenCalledTimes(4)
})
it('deduplicates signalled readers while cancelling only the abandoned subscriber', async () => {
let resolveFetch!: (response: Response) => void
let physicalSignal: AbortSignal | undefined
fetchMock.mockImplementation((_url: unknown, init: RequestInit) => {
physicalSignal = init.signal ?? undefined
return new Promise<Response>(resolve => {
resolveFetch = resolve
})
})
const firstController = new AbortController()
const secondController = new AbortController()
const first = apiGet(
'/api/v1/admin/metadata',
{},
{ ...metadataOptions, signal: firstController.signal }
)
const second = apiGet(
'/api/v1/admin/metadata',
{},
{ ...metadataOptions, signal: secondController.signal }
)
const cancelled = expect(first).rejects.toMatchObject({ name: 'AbortError' })
firstController.abort()
await cancelled
expect(fetchMock).toHaveBeenCalledOnce()
expect(physicalSignal?.aborted).toBe(false)
resolveFetch(success({ stillVisible: true }))
await expect(second).resolves.toEqual({ stillVisible: true })
})
it('starts a fresh GET when a new mount arrives just after the last old subscriber cancelled', async () => {
const old = new AbortController()
fetchMock
.mockImplementationOnce(
(_url: unknown, init: RequestInit) =>
new Promise((_resolve, reject) => {
init.signal!.addEventListener('abort', () =>
reject(new DOMException('Cancelled', 'AbortError'))
)
})
)
.mockResolvedValueOnce(success({ revision: 'fresh-mount' }))
const oldRead = apiGet('/api/v1/admin/metadata', {}, { ...metadataOptions, signal: old.signal })
const rejected = expect(oldRead).rejects.toMatchObject({ name: 'AbortError' })
old.abort()
const freshRead = apiGet(
'/api/v1/admin/metadata',
{},
{ ...metadataOptions, signal: new AbortController().signal }
)
await rejected
await expect(freshRead).resolves.toEqual({ revision: 'fresh-mount' })
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('cancels the physical GET after its last subscriber leaves and cancels an unmounted recovery', async () => {
const controller = new AbortController()
let physicalSignal!: AbortSignal
fetchMock.mockImplementationOnce((_url: unknown, init: RequestInit) => {
physicalSignal = init.signal!
return new Promise((_resolve, reject) => {
physicalSignal.addEventListener('abort', () =>
reject(new DOMException('Cancelled', 'AbortError'))
)
})
})
const pending = apiGet(
'/api/v1/admin/metadata',
{},
{ ...metadataOptions, signal: controller.signal }
)
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' })
controller.abort()
await rejected
expect(physicalSignal.aborted).toBe(true)
const mounted = new AbortController()
fetchMock.mockResolvedValueOnce(throttled(12))
await expect(
apiGet(
'/api/v1/admin/capabilities',
{},
{
...metadataOptions,
signal: mounted.signal,
}
)
).rejects.toMatchObject({ status: 429 })
mounted.abort()
await vi.advanceTimersByTimeAsync(24_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('does not result-cache general reads or automatically replay mutations', async () => {
fetchMock
.mockResolvedValueOnce(success({ revision: 1 }))
.mockResolvedValueOnce(success({ revision: 2 }))
await expect(apiGet('/api/v1/admin/hosts')).resolves.toEqual({ revision: 1 })
await expect(apiGet('/api/v1/admin/hosts')).resolves.toEqual({ revision: 2 })
fetchMock.mockResolvedValueOnce(throttled(12))
await expect(apiSend('POST', '/api/v1/admin/explicit-action')).rejects.toMatchObject({
status: 429,
})
await vi.advanceTimersByTimeAsync(24_000)
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('keeps a read-family cooldown after a successful independent subscription mutation', async () => {
fetchMock.mockResolvedValueOnce(throttled(12))
const denied = await apiGet('/api/v1/admin/capabilities', {}, metadataOptions).catch(
reason => reason
)
fetchMock.mockResolvedValueOnce(success({ models: [] }))
await patchCodexCatalogModel('team-subscription', 'model-one', true)
await expect(
apiGet(
'/api/v1/admin/connections',
{},
{ metadataRead: 'subscription-connections', refresh: true }
)
).rejects.toBe(denied)
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('uses the latest family deadline and keeps recovery while another denied reader is still mounted', async () => {
const firstController = new AbortController()
const secondController = new AbortController()
fetchMock.mockResolvedValueOnce(throttled(12)).mockResolvedValueOnce(throttled(20))
await Promise.all([
apiGet(
'/api/v1/admin/capabilities',
{},
{ ...metadataOptions, signal: firstController.signal }
).catch(() => undefined),
apiGet(
'/api/v1/admin/connections',
{},
{ metadataRead: 'subscription-connections', signal: secondController.signal }
).catch(() => undefined),
])
firstController.abort()
fetchMock
.mockResolvedValueOnce(success({ recovered: true }))
.mockResolvedValueOnce(success({ connections: ['recovered'] }))
await vi.advanceTimersByTimeAsync(12_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
// Both URLs were denied, so the one recovery rereads each of them once.
await vi.advanceTimersByTimeAsync(8_000)
expect(fetchMock).toHaveBeenCalledTimes(4)
await expect(apiGet('/api/v1/admin/capabilities', {}, metadataOptions)).resolves.toEqual({
recovered: true,
})
await expect(
apiGet('/api/v1/admin/connections', {}, { metadataRead: 'subscription-connections' })
).resolves.toEqual({ connections: ['recovered'] })
expect(fetchMock).toHaveBeenCalledTimes(4)
})
it('rereads every URL the server denied while the family recovery was scheduled', async () => {
const codexUrl = '/api/v1/admin/codex-subscription/connections'
const grokUrl = '/api/v1/admin/grok-subscription/connections'
const connectionOptions = { metadataRead: 'subscription-connections' } as const
const pending: Array<(response: Response) => void> = []
fetchMock.mockImplementation(
() =>
new Promise<Response>(resolve => {
pending.push(resolve)
})
)
// Both inventories are in flight before either denial arrives, as when an
// Agent page loads its Codex and Grok connections together.
const codex = apiGet(codexUrl, {}, connectionOptions).catch(reason => reason)
const grok = apiGet(grokUrl, {}, connectionOptions).catch(reason => reason)
await vi.advanceTimersByTimeAsync(0)
expect(pending).toHaveLength(2)
pending[0](throttled(12))
pending[1](throttled(12))
expect(await codex).toMatchObject({ status: 429 })
expect(await grok).toMatchObject({ status: 429 })
fetchMock.mockReset()
fetchMock.mockImplementation((url: string) =>
Promise.resolve(success({ from: url.includes('grok') ? 'grok' : 'codex' }))
)
await vi.advanceTimersByTimeAsync(12_000)
const recoveredUrls = fetchMock.mock.calls.map(([url]) => String(url))
expect(recoveredUrls).toHaveLength(2)
expect(recoveredUrls.filter(url => url.endsWith(codexUrl))).toHaveLength(1)
expect(recoveredUrls.filter(url => url.endsWith(grokUrl))).toHaveLength(1)
// A Retry after the recovery is served from the recovered entries, so it
// cannot spend a third read in the same window.
await expect(apiGet(codexUrl, {}, connectionOptions)).resolves.toEqual({ from: 'codex' })
await expect(apiGet(grokUrl, {}, connectionOptions)).resolves.toEqual({ from: 'grok' })
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('stops the family recovery at the first member the server denies again', async () => {
const codexUrl = '/api/v1/admin/codex-subscription/connections'
const grokUrl = '/api/v1/admin/grok-subscription/connections'
const connectionOptions = { metadataRead: 'subscription-connections' } as const
const pending: Array<(response: Response) => void> = []
fetchMock.mockImplementation(
() =>
new Promise<Response>(resolve => {
pending.push(resolve)
})
)
const codex = apiGet(codexUrl, {}, connectionOptions).catch(reason => reason)
const grok = apiGet(grokUrl, {}, connectionOptions).catch(reason => reason)
await vi.advanceTimersByTimeAsync(0)
expect(pending).toHaveLength(2)
pending[0](throttled(12))
pending[1](throttled(12))
expect(await codex).toMatchObject({ status: 429 })
expect(await grok).toMatchObject({ status: 429 })
fetchMock.mockReset()
fetchMock.mockImplementation(() => Promise.resolve(throttled(12)))
await vi.advanceTimersByTimeAsync(12_000)
// The first reread is denied, which restores the family cooldown; the
// second member is not sent into it, and no further attempt is scheduled.
expect(fetchMock).toHaveBeenCalledOnce()
expect(String(fetchMock.mock.calls[0][0])).toMatch(/codex-subscription\/connections$/)
await vi.advanceTimersByTimeAsync(24_000)
expect(fetchMock).toHaveBeenCalledOnce()
})
it('never sends the remaining members into the cooldown restored by a member denied again', async () => {
// Drives the recovery directly: through apiGet the second member would be
// refused locally by the restored cooldown and send nothing either way.
const familyKey = 'family-under-test'
const denial = Object.assign(new Error('denied again'), { status: 429 })
const firstMember = vi.fn(() => Promise.reject(denial))
const secondMember = vi.fn(() => Promise.resolve(undefined))
const interest = new AbortController()
const deadlineMs = Date.now() + 1_000
expect(
reserveReadRequestRecovery(familyKey, deadlineMs, 'first', firstMember, [interest.signal])
).toBe(true)
expect(
reserveReadRequestRecovery(familyKey, deadlineMs, 'second', secondMember, [interest.signal])
).toBe(false)
await vi.advanceTimersByTimeAsync(1_000)
// Witness: the recovery ran and reread the first member.
expect(firstMember).toHaveBeenCalledOnce()
expect(secondMember).not.toHaveBeenCalled()
// The denied attempt released its reservation for the next denial.
expect(
reserveReadRequestRecovery(familyKey, Date.now() + 1_000, 'first', firstMember, [
interest.signal,
])
).toBe(true)
})
it('gives a waiter its own recovered result when a later member is denied again', async () => {
const codexUrl = '/api/v1/admin/codex-subscription/connections'
const grokUrl = '/api/v1/admin/grok-subscription/connections'
const connectionOptions = { metadataRead: 'subscription-connections' } as const
const pending: Array<(response: Response) => void> = []
fetchMock.mockImplementation(
() =>
new Promise<Response>(resolve => {
pending.push(resolve)
})
)
const codexConsumer = new AbortController()
const grokConsumer = new AbortController()
const codex = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })
const grok = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })
await vi.advanceTimersByTimeAsync(0)
pending.splice(0).forEach(resolve => resolve(throttled(12)))
await expect(codex).rejects.toMatchObject({ status: 429 })
await expect(grok).rejects.toMatchObject({ status: 429 })
await vi.advanceTimersByTimeAsync(12_000)
expect(pending).toHaveLength(1)
const codexWaiter = apiGet(codexUrl, {}, { ...connectionOptions, signal: codexConsumer.signal })
const grokWaiter = apiGet(grokUrl, {}, { ...connectionOptions, signal: grokConsumer.signal })
const codexOutcome = codexWaiter.then(
value => ({ value }),
reason => ({ reason })
)
const grokOutcome = grokWaiter.then(
value => ({ value }),
reason => ({ reason })
)
pending.shift()?.(success({ from: 'codex' }))
await vi.advanceTimersByTimeAsync(0)
expect(pending).toHaveLength(1)
pending.shift()?.(throttled(12))
// Codex was reread before Grok's denial and receives its own result; Grok's
// waiter receives the denial that ended the recovery.
expect(await codexOutcome).toEqual({ value: { from: 'codex' } })
expect(await grokOutcome).toMatchObject({ reason: { status: 429 } })
expect(fetchMock).toHaveBeenCalledTimes(4)
})
it('reads for a waiter without a signal when every registered interest cancels the recovery', async () => {
const pending: Array<(response: Response) => void> = []
fetchMock.mockImplementation(
() =>
new Promise<Response>(resolve => {
pending.push(resolve)
})
)
const mounted = new AbortController()
const first = apiGet(
'/api/v1/admin/metadata',
{},
{ ...metadataOptions, signal: mounted.signal }
)
await vi.advanceTimersByTimeAsync(0)
pending.shift()?.(throttled(2))
await expect(first).rejects.toMatchObject({ status: 429 })
await vi.advanceTimersByTimeAsync(2_000)
expect(pending).toHaveLength(1)
// A consumer such as the feature probes has no signal and cannot be
// registered as an interest.
const unsignalled = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const outcome = unsignalled.then(
value => ({ value }),
reason => ({ reason })
)
mounted.abort()
await vi.advanceTimersByTimeAsync(0)
// The cancelled recovery leaves it a read of its own.
expect(pending).toHaveLength(2)
pending[1](success({ value: 'own read' }))
expect(await outcome).toEqual({ value: { value: 'own read' } })
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('keeps a late recovery scheduled when a URL outside it succeeds', async () => {
const codexUrl = '/api/v1/admin/codex-subscription/connections'
const grokUrl = '/api/v1/admin/grok-subscription/connections'
const connectionOptions = { metadataRead: 'subscription-connections' } as const
const pending = new Map<string, (response: Response) => void>()
fetchMock.mockImplementation(
(input: RequestInfo | URL) =>
new Promise<Response>(resolve => {
pending.set(String(input).includes('grok') ? 'grok' : 'codex', resolve)
})
)
const codex = apiGet(codexUrl, {}, connectionOptions)
const grok = apiGet(grokUrl, {}, connectionOptions)
await vi.advanceTimersByTimeAsync(0)
pending.get('codex')?.(throttled(12))
await expect(codex).rejects.toMatchObject({ status: 429 })
// A background tab runs the recovery timer late: the clock passes the
// deadline before the timer fires, and the slow Grok read succeeds first.
vi.setSystemTime(Date.now() + 13_000)
fetchMock.mockImplementation(() => Promise.resolve(success({ from: 'codex' })))
pending.get('grok')?.(success({ from: 'grok' }))
await expect(grok).resolves.toEqual({ from: 'grok' })
// Fake timers keep their own clock, so the delayed timer fires only now.
await vi.advanceTimersByTimeAsync(12_000)
// The Codex member is still reread.
expect(fetchMock).toHaveBeenCalledTimes(3)
expect(String(fetchMock.mock.calls[2][0])).toMatch(/codex-subscription\/connections$/)
await expect(apiGet(codexUrl, {}, connectionOptions)).resolves.toEqual({ from: 'codex' })
expect(fetchMock).toHaveBeenCalledTimes(3)
})
it('moves the retry time of every earlier denial when a later one extends the cooldown', () => {
const familyKey = 'family-under-test'
const startedAt = Date.now()
const earlier = Object.assign(new Error('earlier'), { status: 429 })
const later = Object.assign(new Error('later'), { status: 429 })
expect(setReadRequestCooldown(familyKey, earlier, 12, startedAt)).toBe(12_000)
expect(setReadRequestCooldown(familyKey, later, 12, startedAt + 500)).toBe(12_000)
expect(later.retryAtMs).toBe(startedAt + 12_500)
expect(earlier.retryAtMs).toBe(startedAt + 12_500)
// A denial after the cooldown expired starts a new window and leaves the
// errors of the expired one alone.
const next = Object.assign(new Error('next'), { status: 429 })
setReadRequestCooldown(familyKey, next, 12, startedAt + 20_000)
expect(next.retryAtMs).toBe(startedAt + 32_000)
expect(earlier.retryAtMs).toBe(startedAt + 12_500)
})
it('schedules a retry at the extended time instead of the time first issued', async () => {
const familyKey = 'family-under-test'
const startedAt = Date.now()
const earlier = Object.assign(new Error('earlier'), { status: 429 })
setReadRequestCooldown(familyKey, earlier, 12, startedAt)
const retry = vi.fn()
scheduleReadRequestRetry(earlier, retry)
await vi.advanceTimersByTimeAsync(500)
setReadRequestCooldown(
familyKey,
Object.assign(new Error('later'), { status: 429 }),
12,
startedAt + 500
)
await vi.advanceTimersByTimeAsync(11_500)
expect(retry).not.toHaveBeenCalled()
// Witness: the retry runs once the extended time is reached.
await vi.advanceTimersByTimeAsync(500)
expect(retry).toHaveBeenCalledOnce()
})
it('waits for the family recovery already rereading before it re-runs the consumer', async () => {
const path = '/api/v1/admin/connections'
const connections = { metadataRead: 'subscription-connections' } as const
// The recovery's reread stays in flight until released, and rejects on
// abort the way a browser fetch does.
let releaseReread!: () => void
let rereadAborted = false
fetchMock.mockResolvedValueOnce(throttled(12)).mockImplementationOnce(
(_input: RequestInfo | URL, init?: RequestInit) =>
new Promise<Response>((resolve, reject) => {
releaseReread = () => resolve(success({ connections: ['recovered'] }))
init?.signal?.addEventListener('abort', () => {
rereadAborted = true
reject(new DOMException('The read was cancelled', 'AbortError'))
})
})
)
// The consumer's effect: its cleanup aborts the signal registered as the
// recovery's interest, then the effect reads again.
const effect = new AbortController()
let denial: unknown
await apiGet(path, {}, { ...connections, signal: effect.signal }).catch(error => {
denial = error
})
expect(denial).toMatchObject({ status: 429 })
const rerun = vi.fn(() => {
effect.abort()
return apiGet(path, {}, connections)
})
// Scheduled after the recovery, as the consumer does, so at the shared
// deadline the recovery's timer fires first and its reread is in flight
// when this one fires.
scheduleReadRequestRetry(denial as Parameters<typeof scheduleReadRequestRetry>[0], rerun)
await vi.advanceTimersByTimeAsync(12_000)
// Witness: the recovery is rereading at the deadline.
expect(fetchMock).toHaveBeenCalledTimes(2)
expect(rerun).not.toHaveBeenCalled()
expect(rereadAborted).toBe(false)
releaseReread()
await vi.advanceTimersByTimeAsync(0)
// The consumer re-runs once the recovery settled, and the recovered entry
// serves its read.
expect(rerun).toHaveBeenCalledOnce()
await expect(rerun.mock.results[0]?.value).resolves.toEqual({ connections: ['recovered'] })
expect(rereadAborted).toBe(false)
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('changes the session identity only when the confirmed principal changes', () => {
const start = getReadRequestSessionIdentity()
setReadRequestCacheEntry('entry', { value: 1 }, 30_000)
invalidateReadRequestCache()
invalidateReadRequestCacheEntry('entry')
setControlUIReadPrincipal('admin-one', 'admin')
expect(getReadRequestSessionIdentity()).toBe(start)
// Witness: a different principal does change it.
setControlUIReadPrincipal('admin-two', 'admin')
expect(getReadRequestSessionIdentity()).toBe(start + 1)
})
it('makes only one recovery when throttle timing is unavailable', async () => {
const noTiming = () =>
new Response(JSON.stringify({ error: 'Too Many Requests' }), { status: 429 })
fetchMock.mockImplementation(() => Promise.resolve(noTiming()))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).rejects.toMatchObject({
status: 429,
})
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('schedules one recovery for a later denial after a recovery was itself denied', async () => {
const path = '/api/v1/admin/connections'
const connections = { metadataRead: 'subscription-connections' } as const
fetchMock.mockResolvedValueOnce(throttled(12)).mockResolvedValueOnce(throttled(12))
await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })
await vi.advanceTimersByTimeAsync(12_000)
// The single recovery ran and was denied; it does not schedule another.
expect(fetchMock).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
fetchMock
.mockResolvedValueOnce(throttled(12))
.mockResolvedValueOnce(success({ connections: ['recovered'] }))
await expect(apiGet(path, {}, connections)).rejects.toMatchObject({ status: 429 })
expect(fetchMock).toHaveBeenCalledTimes(3)
await vi.advanceTimersByTimeAsync(12_000)
expect(fetchMock).toHaveBeenCalledTimes(4)
expect(String(fetchMock.mock.calls[3][0])).toContain(path)
await expect(apiGet(path, {}, connections)).resolves.toEqual({ connections: ['recovered'] })
expect(fetchMock).toHaveBeenCalledTimes(4)
})
describe('consumers refused by the family cooldown', () => {
const connections = '/api/v1/admin/connections'
const models = '/api/v1/admin/models'
async function denyFirstConsumer(signal: AbortSignal) {
fetchMock.mockResolvedValueOnce(throttled(2))
await expect(
apiGet(connections, {}, { metadataRead: 'subscription-connections', signal })
).rejects.toMatchObject({ status: 429 })
}
it.each([
['the same URL', connections],
['a different URL in the same family', models],
])(
'keep the recovery alive after the first consumer leaves (%s)',
async (_label, joinedPath) => {
const first = new AbortController()
const second = new AbortController()
await denyFirstConsumer(first.signal)
await vi.advanceTimersByTimeAsync(1_000)
await expect(
apiGet(
joinedPath,
{},
{ metadataRead: 'subscription-model-catalog', signal: second.signal }
)
).rejects.toMatchObject({ status: 429 })
// Joining sends nothing.
expect(fetchMock).toHaveBeenCalledOnce()
first.abort()
fetchMock.mockResolvedValueOnce(success({ connections: ['recovered'] }))
// The original deadline (t = 2 s) still applies: joining at t = 1 s
// neither extended it nor added a second attempt.
await vi.advanceTimersByTimeAsync(1_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
expect(String(fetchMock.mock.calls[1][0])).toMatch(/\/api\/v1\/admin\/connections$/)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledTimes(2)
}
)
it('cancel the recovery once every registered consumer has left', async () => {
const first = new AbortController()
const second = new AbortController()
await denyFirstConsumer(first.signal)
await expect(
apiGet(models, {}, { metadataRead: 'subscription-model-catalog', signal: second.signal })
).rejects.toMatchObject({ status: 429 })
// Witness: the denial scheduled the recovery, so "no reread" below
// means it was cancelled, not that it never existed.
expect(vi.getTimerCount()).toBe(1)
first.abort()
expect(vi.getTimerCount()).toBe(1)
second.abort()
expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledOnce()
})
it('do not hold the recovery alive when they have no signal to observe', async () => {
const first = new AbortController()
await denyFirstConsumer(first.signal)
await expect(
apiGet(models, {}, { metadataRead: 'subscription-model-catalog' })
).rejects.toMatchObject({ status: 429 })
expect(fetchMock).toHaveBeenCalledOnce()
// Witness: the recovery was scheduled before the only registered
// consumer left.
expect(vi.getTimerCount()).toBe(1)
first.abort()
expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledOnce()
})
})
it('keeps a running recovery alive for a consumer waiting on it after the deadline', async () => {
const path = '/api/v1/admin/connections'
const connectionOptions = { metadataRead: 'subscription-connections' } as const
const first = new AbortController()
const waiter = new AbortController()
fetchMock.mockResolvedValueOnce(throttled(2))
await expect(
apiGet(path, {}, { ...connectionOptions, signal: first.signal })
).rejects.toMatchObject({ status: 429 })
let finishReread!: (response: Response) => void
fetchMock.mockImplementationOnce(
() =>
new Promise<Response>(resolve => {
finishReread = resolve
})
)
await vi.advanceTimersByTimeAsync(2_000)
// The recovery is running: its reread is in flight.
expect(fetchMock).toHaveBeenCalledTimes(2)
const waiting = apiGet(path, {}, { ...connectionOptions, signal: waiter.signal })
await vi.advanceTimersByTimeAsync(0)
// The consumer that scheduled the recovery leaves while another waits on it.
first.abort()
finishReread(success({ connections: ['recovered'] }))
await expect(waiting).resolves.toEqual({ connections: ['recovered'] })
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('schedules a recovery for a remount after every earlier subscriber left', async () => {
const path = '/api/v1/admin/connections'
const firstMount = new AbortController()
fetchMock.mockResolvedValueOnce(throttled(12))
await expect(
apiGet(path, {}, { metadataRead: 'subscription-connections', signal: firstMount.signal })
).rejects.toMatchObject({ status: 429 })
firstMount.abort()
await vi.advanceTimersByTimeAsync(24_000)
expect(fetchMock).toHaveBeenCalledTimes(1)
const secondMount = new AbortController()
fetchMock
.mockResolvedValueOnce(throttled(12))
.mockResolvedValueOnce(success({ connections: ['remounted'] }))
await expect(
apiGet(path, {}, { metadataRead: 'subscription-connections', signal: secondMount.signal })
).rejects.toMatchObject({ status: 429 })
expect(fetchMock).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(12_000)
expect(fetchMock).toHaveBeenCalledTimes(3)
await expect(apiGet(path, {}, { metadataRead: 'subscription-connections' })).resolves.toEqual({
connections: ['remounted'],
})
})
it('clears metadata on an epoch change and broadcasts only a non-sensitive invalidation', async () => {
const listener = new BroadcastChannel('control-ui-read-metadata-invalidation')
const received: unknown[] = []
listener.onmessage = event => received.push(event.data)
const sends = vi.spyOn(BroadcastChannel.prototype, 'postMessage')
try {
fetchMock.mockResolvedValueOnce(success({ principal: 'one' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
principal: 'one',
})
clearAdminAuthToken()
expect(sends).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(received).toEqual([{ type: 'session-invalidation' }]))
expect(JSON.stringify(received)).not.toContain('admin-one')
setControlUIReadPrincipal('admin-one', 'admin')
setControlUIReadPrincipal('admin-two', 'admin')
expect(sends).toHaveBeenCalledTimes(2)
await vi.waitFor(() =>
expect(received).toEqual([
{ type: 'session-invalidation' },
{ type: 'session-invalidation' },
])
)
fetchMock.mockResolvedValueOnce(success({ principal: 'two' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
principal: 'two',
})
} finally {
listener.close()
}
})
it('invalidates on a remote session change without rebroadcast or caching an unverified identity', async () => {
const remoteTab = new BroadcastChannel('control-ui-read-metadata-invalidation')
try {
fetchMock.mockResolvedValueOnce(success({ revision: 'old-tab' }))
await apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const sends = vi.spyOn(BroadcastChannel.prototype, 'postMessage')
remoteTab.postMessage({ type: 'session-invalidation' })
// Witness: the message was delivered and this tab dropped its principal.
await vi.waitFor(() => expect(getReadRequestPrincipal()).toBeNull())
expect(sends.mock.contexts.filter(channel => channel !== remoteTab)).toHaveLength(0)
fetchMock
.mockResolvedValueOnce(success({ revision: 'new-tab' }))
.mockResolvedValueOnce(success({ revision: 'fresh' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 'new-tab',
})
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 'fresh',
})
expect(fetchMock).toHaveBeenCalledTimes(3)
} finally {
remoteTab.close()
}
})
it('answers the post-invalidation 401 locally so logged-out tabs cannot invalidate each other forever', async () => {
const remoteTab = new BroadcastChannel('control-ui-read-metadata-invalidation')
const received: unknown[] = []
remoteTab.onmessage = event => received.push(event.data)
const sends = vi.spyOn(BroadcastChannel.prototype, 'postMessage')
const sentByThisTab = () => sends.mock.contexts.filter(channel => channel !== remoteTab).length
try {
// The other tab logged out: this tab drops its principal on the message.
remoteTab.postMessage({ type: 'session-invalidation' })
await vi.waitFor(() => expect(getReadRequestPrincipal()).toBeNull())
// Its /auth/me re-check now answers 401, which clears the token again.
setReadRequestCacheEntry('post-invalidation-entry', { stale: true }, 30_000)
clearAdminAuthToken()
// Witness: the local clear ran and dropped the entry cached after the message.
expect(getReadRequestCacheEntry('post-invalidation-entry')).toBeUndefined()
expect(sentByThisTab()).toBe(0)
// A verified tab that logs out still tells the others exactly once.
setControlUIReadPrincipal('admin-one', 'admin')
clearAdminAuthToken()
expect(sentByThisTab()).toBe(1)
await vi.waitFor(() => expect(received).toEqual([{ type: 'session-invalidation' }]))
} finally {
remoteTab.close()
}
})
it('tells peer tabs about the committed cookie change after a login or logout POST', async () => {
const peerTab = new BroadcastChannel('control-ui-read-metadata-invalidation')
const received: unknown[] = []
peerTab.onmessage = event => received.push(event.data)
const sends = vi.spyOn(BroadcastChannel.prototype, 'postMessage')
const invalidation = { type: 'session-invalidation' }
try {
let commitLogin: (response: Response) => void = () => {}
fetchMock.mockReturnValueOnce(new Promise<Response>(resolve => (commitLogin = resolve)))
const login = loginControlUI('operator', 'unit-test-password')
// Pre-POST clear of the verified principal; a peer may now reconfirm the old cookie.
expect(sends).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(received).toEqual([invalidation]))
commitLogin(success({ me: { id: 'admin-two', role: 'admin' } }))
await login
expect(sends).toHaveBeenCalledTimes(2)
await vi.waitFor(() => expect(received).toEqual([invalidation, invalidation]))
setControlUIReadPrincipal('admin-two', 'admin')
sends.mockClear()
received.length = 0
let commitLogout: (response: Response) => void = () => {}
fetchMock.mockReturnValueOnce(new Promise<Response>(resolve => (commitLogout = resolve)))
const logout = logoutControlUI()
expect(sends).toHaveBeenCalledTimes(1)
await vi.waitFor(() => expect(received).toEqual([invalidation]))
commitLogout(success({ ok: true }))
await logout
expect(sends).toHaveBeenCalledTimes(2)
await vi.waitFor(() => expect(received).toEqual([invalidation, invalidation]))
expect(fetchMock).toHaveBeenCalledTimes(2)
} finally {
peerTab.close()
}
})
it('clears cached reads on successful login even when the principal is unchanged', async () => {
fetchMock.mockResolvedValueOnce(success({ revision: 1 }))
await apiGet('/api/v1/admin/metadata', {}, metadataOptions)
fetchMock.mockResolvedValueOnce(success({ me: { id: 'admin-one', role: 'admin' } }))
await loginControlUI('operator', 'unit-test-password')
setControlUIReadPrincipal('admin-one', 'admin')
fetchMock.mockResolvedValueOnce(success({ revision: 2 }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 2,
})
})
it('invalidates a scope change for the same verified principal', async () => {
fetchMock.mockResolvedValueOnce(success({ scope: 'first' }))
await apiGet('/api/v1/admin/metadata', {}, metadataOptions)
setControlUIReadPrincipal('admin-one', 'admin:second-scope')
fetchMock.mockResolvedValueOnce(success({ scope: 'second' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
scope: 'second',
})
expect(fetchMock).toHaveBeenCalledTimes(2)
})
it('binds browser metadata to the API origin instead of reusing another origin response', async () => {
fetchMock.mockResolvedValueOnce(success({ origin: 'first' }))
await apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const originalWindow = globalThis.window
vi.stubGlobal('window', { location: { origin: 'https://other-control.example.test' } })
try {
fetchMock.mockResolvedValueOnce(success({ origin: 'second' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
origin: 'second',
})
expect(fetchMock).toHaveBeenCalledTimes(2)
} finally {
vi.stubGlobal('window', originalWindow)
}
})
it('fences old errors and 401 responses before they can invalidate the new principal', async () => {
let resolveFetch!: (response: Response) => void
fetchMock.mockImplementationOnce(
() =>
new Promise<Response>(resolve => {
resolveFetch = resolve
})
)
const authExpired = vi.fn()
setGlobalAuthErrorHandler(authExpired)
const oldRead = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const rejected = expect(oldRead).rejects.toMatchObject({ name: 'AuthExpiredError' })
setControlUIReadPrincipal('admin-two', 'admin')
resolveFetch(new Response('{}', { status: 401 }))
await rejected
expect(authExpired).not.toHaveBeenCalled()
expect(getReadRequestPrincipal()?.principalId).toBe('admin-two')
})
it('fences a delayed response body and a metadata mutation before result delivery', async () => {
let resolveBody!: (value: string) => void
const response = success({})
vi.spyOn(response, 'text').mockImplementation(
() =>
new Promise(resolve => {
resolveBody = resolve
})
)
fetchMock.mockResolvedValueOnce(response)
const pending = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
await vi.advanceTimersByTimeAsync(0)
invalidateReadRequestCache()
const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' })
resolveBody(JSON.stringify({ revision: 'before-mutation' }))
await rejected
fetchMock.mockResolvedValueOnce(success({ revision: 'after-mutation' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
revision: 'after-mutation',
})
})
it('rejects a late old-principal response before caching or delivering it to the new identity', async () => {
let resolveFetch!: (response: Response) => void
fetchMock.mockReturnValue(
new Promise<Response>(resolve => {
resolveFetch = resolve
})
)
const staleRead = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
await vi.advanceTimersByTimeAsync(0)
expect(fetchMock).toHaveBeenCalledOnce()
setControlUIReadPrincipal('admin-two', 'admin')
resolveFetch(success({ principal: 'one' }))
await expect(staleRead).rejects.toMatchObject({ name: 'AuthExpiredError' })
fetchMock.mockResolvedValueOnce(success({ principal: 'two' }))
await expect(apiGet('/api/v1/admin/metadata', {}, metadataOptions)).resolves.toEqual({
principal: 'two',
})
})
it('does not retain or deduplicate opt-in metadata during SSR execution', async () => {
const originalWindow = globalThis.window
vi.stubGlobal('window', undefined)
try {
setReadRequestCacheEntry('ssr-key', { value: 'must-not-cache' }, 30_000)
expect(getReadRequestCacheEntry('ssr-key')).toBeUndefined()
fetchMock
.mockResolvedValueOnce(success({ principal: 'server-one' }))
.mockResolvedValueOnce(success({ principal: 'server-two' }))
const first = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
const second = apiGet('/api/v1/admin/metadata', {}, metadataOptions)
await expect(first).resolves.toEqual({ principal: 'server-one' })
await expect(second).resolves.toEqual({ principal: 'server-two' })
expect(fetchMock).toHaveBeenCalledTimes(2)
} finally {
vi.stubGlobal('window', originalWindow)
}
})
})