Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-ui): keep a running read recovery alive for consumers waiting on it

Alfredo Lopez707a4b4cc55b
diff --git a/control-ui/lib/__tests__/readRequestCache.test.ts b/control-ui/lib/__tests__/readRequestCache.test.ts
index ff28a887f..bc412e28b 100644
--- a/control-ui/lib/__tests__/readRequestCache.test.ts
+++ b/control-ui/lib/__tests__/readRequestCache.test.ts
@@ -516,8 +516,13 @@ describe('bounded metadata read reuse', () => {
await expect(
apiGet(models, {}, { metadataRead: 'subscription-model-catalog', signal: second.signal })
).rejects.toMatchObject({ status: 429 })
+ // Witness: the denial scheduled the recovery, so "no reread" below
+ // means it was cancelled, not that it never existed.
+ expect(vi.getTimerCount()).toBe(1)
first.abort()
+ expect(vi.getTimerCount()).toBe(1)
second.abort()
+ expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledOnce()
})
@@ -529,12 +534,45 @@ describe('bounded metadata read reuse', () => {
apiGet(models, {}, { metadataRead: 'subscription-model-catalog' })
).rejects.toMatchObject({ status: 429 })
expect(fetchMock).toHaveBeenCalledOnce()
+ // Witness: the recovery was scheduled before the only registered
+ // consumer left.
+ expect(vi.getTimerCount()).toBe(1)
first.abort()
+ expect(vi.getTimerCount()).toBe(0)
await vi.advanceTimersByTimeAsync(60_000)
expect(fetchMock).toHaveBeenCalledOnce()
})
})
+ it('keeps a running recovery alive for a consumer waiting on it after the deadline', async () => {
+ const path = '/api/v1/admin/connections'
+ const connectionOptions = { metadataRead: 'subscription-connections' } as const
+ const first = new AbortController()
+ const waiter = new AbortController()
+ fetchMock.mockResolvedValueOnce(throttled(2))
+ await expect(
+ apiGet(path, {}, { ...connectionOptions, signal: first.signal })
+ ).rejects.toMatchObject({ status: 429 })
+ let finishReread!: (response: Response) => void
+ fetchMock.mockImplementationOnce(
+ () =>
+ new Promise<Response>(resolve => {
+ finishReread = resolve
+ })
+ )
+ await vi.advanceTimersByTimeAsync(2_000)
+ // The recovery is running: its reread is in flight.
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+
+ const waiting = apiGet(path, {}, { ...connectionOptions, signal: waiter.signal })
+ await vi.advanceTimersByTimeAsync(0)
+ // The consumer that scheduled the recovery leaves while another waits on it.
+ first.abort()
+ finishReread(success({ connections: ['recovered'] }))
+ await expect(waiting).resolves.toEqual({ connections: ['recovered'] })
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ })
+
it('schedules a recovery for a remount after every earlier subscriber left', async () => {
const path = '/api/v1/admin/connections'
const firstMount = new AbortController()
diff --git a/control-ui/lib/api.ts b/control-ui/lib/api.ts
index 56cd02993..ddbeda0e2 100644
--- a/control-ui/lib/api.ts
+++ b/control-ui/lib/api.ts
@@ -457,6 +457,7 @@ async function apiRead(
if (!recoveringFamily) {
const recovery = getReadRequestRecovery(familyKey)
if (recovery) {
+ joinReadRequestRecovery(familyKey, options.signal)
await waitForRead(recovery, options.signal)
assertCurrent()
return apiRead(path, query, options)
diff --git a/control-ui/lib/readRequestCache.ts b/control-ui/lib/readRequestCache.ts
index ca3820164..7eb253e2a 100644
--- a/control-ui/lib/readRequestCache.ts
+++ b/control-ui/lib/readRequestCache.ts
@@ -227,17 +227,17 @@ function disposeRecovery(entry: RecoveryEntry): void {
}
/**
- * Register a consumer that the family cooldown refused as an interest in the
- * scheduled recovery, so another consumer unmounting cannot cancel it while
- * this one is still mounted. It sends nothing and leaves the deadline, the
- * reread members and the attempt budget unchanged. A consumer without a
- * signal has no lifecycle to observe and is not registered, so it can never
- * hold a recovery alive on its own.
+ * Register a consumer as an interest in the family's recovery, so another
+ * consumer unmounting cannot cancel it while this one is still mounted: one
+ * refused by the cooldown while the recovery is scheduled, or one waiting on
+ * it after the deadline. It sends nothing and leaves the deadline, the reread
+ * members and the attempt budget unchanged. A consumer without a signal has no
+ * lifecycle to observe and is not registered, so it can never hold a recovery
+ * alive on its own.
*/
export function joinReadRequestRecovery(key: string, signal: AbortSignal | undefined): void {
if (!signal || signal.aborted) return
- const recovery = recoveries.get(key)
- if (recovery?.state === 'scheduled') recovery.addSubscribers([signal])
+ recoveries.get(key)?.addSubscribers([signal])
}
export function getReadRequestRecovery(key: string): Promise<void> | undefined {
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index c82016381..a9403fe83 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.715",
+ "version": "0.1.716",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.715",
+ "version": "0.1.716",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 0dc0b151f..7a1d97d58 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.715",
+ "version": "0.1.716",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/docs/architecture/administrative-rate-limits.md b/docs/architecture/administrative-rate-limits.md
index c2dfd8242..9bf13301a 100644
--- a/docs/architecture/administrative-rate-limits.md
+++ b/docs/architecture/administrative-rate-limits.md
@@ -155,9 +155,10 @@ once each and one at a time, every URL the server denied while it was
scheduled (for example the Codex and Grok inventories loaded together), and
stops at the first reread that is denied again. A denied recovery schedules no
further attempt, but it releases its reservation, so the next denial in that
-read family schedules its own. A consumer refused by the cooldown joins the
-scheduled recovery as an interest without sending a request or changing its
-deadline, members or budget, so the recovery is cancelled only when every
+read family schedules its own. A consumer refused by the cooldown, or waiting
+on the recovery after its deadline, joins it as an interest without sending a
+request or changing its deadline, members or budget, so the recovery is
+cancelled only when every
registered consumer has left; a consumer without an abort signal is not
registered. Capability consumers follow the confirmed principal: when it
changes they drop the previous session's result and error and load for the