Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(proxy): admit messages during stateless wake

Alfredo Lopezd09578f1be37
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index fdc83883b..48d93c83f 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.9.0',
externalRestApiVersion: '0.1.84',
- rpcProxyVersion: '0.1.90',
+ rpcProxyVersion: '0.1.91',
desktopVersion: '0.9.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index 1e5967806..24d635091 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.90",
+ "version": "0.1.91",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.90",
+ "version": "0.1.91",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index cf15847a3..6b775fd3f 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.90",
+ "version": "0.1.91",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
index bf37f6bf4..0fd4d72af 100644
--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import request from 'supertest'
import { createRpcRouter } from '../routes/rpc.js'
@@ -96,6 +96,10 @@ beforeEach(() => {
serviceMock.resolveHostConnectionForUser.mockResolvedValue({ ...HOST_CONNECTION })
})
+afterEach(() => {
+ vi.useRealTimers()
+})
+
describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
it('host-down network error triggers a wake; 200 active leads to one immediate upstream retry', async () => {
serviceMock.forwardHostMessageToHost
@@ -114,6 +118,51 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)
})
+ it('starts wake after a short availability probe and admits after the Host becomes ready', async () => {
+ let firstForwardStarted!: () => void
+ const firstForward = new Promise<void>(resolve => {
+ firstForwardStarted = resolve
+ })
+ let upstreamAdmits = false
+ const availabilityTimeout = new Error('Initial host availability probe timed out')
+ availabilityTimeout.name = 'HostAvailabilityProbeTimeoutError'
+ serviceMock.forwardHostMessageToHost
+ .mockImplementationOnce(
+ (_host: unknown, _body: unknown, options: { timeoutMs?: number }) =>
+ new Promise((_resolve, reject) => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
+ expect(options.timeoutMs).toBe(750)
+ firstForwardStarted()
+ setTimeout(() => reject(availabilityTimeout), options.timeoutMs)
+ })
+ )
+ .mockImplementation(async () => {
+ if (!upstreamAdmits) throw hostDownError()
+ return { success: true, taskId: 't-delayed' }
+ })
+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({
+ kind: 'active',
+ wakeGeneration: null,
+ })
+
+ const responsePromise = postMessage(makeApp())
+ .expect(200)
+ .then(response => response)
+ await firstForward
+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(750)
+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)
+ expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(1_500)
+ upstreamAdmits = true
+ await vi.advanceTimersByTimeAsync(250)
+ const response = await responsePromise
+ expect(response.body).toEqual({ success: true, taskId: 't-delayed' })
+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][1]).toBe(
+ serviceMock.forwardHostMessageToHost.mock.calls.at(-1)![1]
+ )
+ })
+
it('the wake retry re-forwards the SAME stable messageId as the first forward (idempotency identity)', async () => {
// P1-1: without a stable delivery id the wake-and-hold retry re-forwards
// the same turn under a fresh uuid at mcp-host and executes it twice. The
@@ -277,17 +326,26 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
})
it('host still unreachable after a wake reports active resolves to host_waking, never a hang', async () => {
+ let wakeStarted!: () => void
+ const wakeCalled = new Promise<void>(resolve => {
+ wakeStarted = resolve
+ })
serviceMock.forwardHostMessageToHost.mockRejectedValue(hostDownError())
- controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({
- kind: 'active',
- wakeGeneration: null,
+ controlApiMock.requestHostWakeFromControlApi.mockImplementation(async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
+ wakeStarted()
+ return { kind: 'active', wakeGeneration: null }
})
- const response = await postMessage(makeApp()).expect(503)
+ const responsePromise = postMessage(makeApp())
+ .expect(503)
+ .then(response => response)
+ await wakeCalled
+ await vi.advanceTimersByTimeAsync(48_000)
+ const response = await responsePromise
expect(response.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })
- // Trigger + the full short retry schedule were attempted.
- expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(4)
+ expect(serviceMock.forwardHostMessageToHost.mock.calls.length).toBeGreaterThan(4)
})
it('a non-availability upstream failure (mcp-host 500) never triggers a wake', async () => {
@@ -311,6 +369,22 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {
expect(response.body).toEqual({ error: 'Gateway Timeout' })
expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
})
+
+ it('a token without wake scope keeps the original full upstream timeout', async () => {
+ authTokenMock.verifyRpcToken.mockReturnValue({
+ ...VALID_CLAIMS,
+ scopes: ['host:message:invoke'],
+ })
+ const abort = new Error('aborted')
+ abort.name = 'AbortError'
+ serviceMock.forwardHostMessageToHost.mockRejectedValue(abort)
+
+ await postMessage(makeApp()).expect(504)
+
+ expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(1)
+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][2]).toEqual({ async: false })
+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()
+ })
})
describe('task result and cancel host-down paths', () => {
diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.ts
index b7027712b..829721aec 100644
--- a/rpc-proxy/src/routes/rpc.ts
+++ b/rpc-proxy/src/routes/rpc.ts
@@ -32,7 +32,9 @@ import {
validateRpcRequest,
} from '../services/mcpProxyService.js'
import {
+ MAX_REQUEST_HOLD_MS,
isUpstreamTimeoutError,
+ isWakeCapable,
isWakeEligibleHostError,
respondWithWakeAndHold,
} from '../services/wakeAndHold.js'
@@ -45,6 +47,8 @@ const RFC1123_RE = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/
// own copy (the two services share no package, so each names its own).
const SESSIONS_LIMIT_CAP = 100
const MESSAGES_LIMIT_CAP = 200
+/** A suspended pod may spend seconds failing DNS/connect before the wake starts. */
+const MESSAGE_AVAILABILITY_PROBE_MS = 750
function isSafeUpstreamPathSegment(value: string): boolean {
return (
@@ -403,6 +407,7 @@ export function createRpcRouter(): Router {
: {}),
}
+ const wakeDeadlineMs = Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)
const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {
teamId: auth.teamId,
...(traceContext.sessionId
@@ -437,9 +442,11 @@ export function createRpcRouter(): Router {
`[RPC_PROXY] user=${auth.sub} host=${hostRef} method=host-message-rest${isAsync ? ' (async)' : ''} attachments=${attachmentCount}`
)
let upstreamResponse: Record<string, unknown> | null = null
+ const wakeCapable = isWakeCapable(auth, hostRef, Date.now())
try {
upstreamResponse = await forwardHostMessageToHost(host, forwardedBody, {
async: isAsync,
+ ...(wakeCapable ? { timeoutMs: MESSAGE_AVAILABILITY_PROBE_MS } : {}),
})
} catch (error) {
console.warn(
@@ -447,7 +454,12 @@ export function createRpcRouter(): Router {
error instanceof Error ? error.message : String(error)
}`
)
- if (isWakeEligibleHostError(error)) {
+ if (
+ isWakeEligibleHostError(error) ||
+ (wakeCapable &&
+ error instanceof Error &&
+ error.name === 'HostAvailabilityProbeTimeoutError')
+ ) {
// Stateless wake-and-hold: a down or draining host triggers a
// control-api wake (and possibly a bounded hold) instead of the
// generic 502. Non-stateless hosts fall back to the legacy path.
@@ -457,6 +469,7 @@ export function createRpcRouter(): Router {
host,
claims: auth,
rpcAccessToken,
+ deadlineMs: wakeDeadlineMs,
attemptUpstream: async () => {
const retried = await forwardHostMessageToHost(host, forwardedBody, {
async: isAsync,
diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
index 795f85994..5f6ac615d 100644
--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts
@@ -1,5 +1,36 @@
-import { describe, expect, it } from 'vitest'
-import { __test__normalizeHostStatusPayload as normalize } from '../mcpHostRestService.js'
+import { describe, expect, it, vi } from 'vitest'
+import {
+ forwardHostMessageToHost,
+ __test__normalizeHostStatusPayload as normalize,
+} from '../mcpHostRestService.js'
+
+describe('host message availability probe', () => {
+ it('aborts at the requested short timeout with a distinct wake-eligible reason', async () => {
+ vi.useFakeTimers()
+ const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(
+ (_input, init) =>
+ new Promise((_resolve, reject) => {
+ init?.signal?.addEventListener('abort', () => reject(init.signal?.reason))
+ })
+ )
+ try {
+ const pending = forwardHostMessageToHost(
+ { name: 'chatllm', url: 'http://chatllm:8080', headers: {} },
+ { content: 'hello', hostRef: 'chatllm' },
+ { timeoutMs: 750 }
+ )
+ const rejection = expect(pending).rejects.toMatchObject({
+ name: 'HostAvailabilityProbeTimeoutError',
+ })
+ await vi.advanceTimersByTimeAsync(750)
+ await rejection
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ } finally {
+ fetchMock.mockRestore()
+ vi.useRealTimers()
+ }
+ })
+})
const baseUpstream = {
agent: {
diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
index 0758e8c7f..da48afd64 100644
--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts
@@ -559,7 +559,7 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
expect(requestWake.mock.calls.length).toBe(wakeCalls)
})
- it('(c) an unresolved proceed still walks the short retry schedule exactly as today', async () => {
+ it('(c) an unresolved proceed retries availability until admission succeeds', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
@@ -573,14 +573,14 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
- await vi.advanceTimersByTimeAsync(1_250) // 0ms + 250ms + 1000ms schedule
+ await vi.advanceTimersByTimeAsync(750) // 0ms + 250ms + 750ms attempts
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(3) // pre-resolution retries are the feature
expect(res.statusCode).toBe(200)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
- it('active wake response currently returns host_waking when upstream admission becomes possible after the bounded retry schedule', async () => {
+ it('active wake response keeps trying until late upstream admission succeeds within the hold', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
@@ -592,22 +592,38 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
- await vi.advanceTimersByTimeAsync(1_250) // exhaust 0ms + 250ms + 1000ms attempts
- await pending
+ await vi.advanceTimersByTimeAsync(1_250)
expect(attemptUpstream).toHaveBeenCalledTimes(3)
- expect(res.statusCode).toBe(503)
- expect(res.body).toMatchObject({
- code: 'host_waking',
- hostRef: 'chatllm',
- lastKnownState: 'active',
- })
+ expect(res.headersSent).toBe(false)
expect(probeReady).not.toHaveBeenCalled()
expect(coordinator.trackedCoordinationCount()).toBe(0)
upstreamAdmits = true
+ await vi.advanceTimersByTimeAsync(500)
+ await pending
+ expect(attemptUpstream).toHaveBeenCalledTimes(4)
+ expect(res.statusCode).toBe(200)
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)
- expect(attemptUpstream).toHaveBeenCalledTimes(3) // admission later does not trigger a retry
+ expect(attemptUpstream).toHaveBeenCalledTimes(4)
+ })
+
+ it('returns host_waking at the absolute deadline without a later admission attempt', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
+ const res = makeRes()
+ const attemptUpstream = vi.fn().mockRejectedValue(upstreamDrainingError())
+ const pending = respondWithWakeAndHold({
+ ...respondOptions(coordinator, res, attemptUpstream),
+ deadlineMs: Date.now() + 1_000,
+ })
+
+ await vi.advanceTimersByTimeAsync(1_000)
+ await pending
+ expect(attemptUpstream).toHaveBeenCalledTimes(3)
expect(res.statusCode).toBe(503)
+ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })
+ await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
+ expect(attemptUpstream).toHaveBeenCalledTimes(3)
})
it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {
@@ -704,6 +720,31 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
+ it('suppresses a hold outcome after another path commits the response', async () => {
+ const { coordinator, requestWake } = makeCoordinator()
+ let finishWake!: (value: HostWakeApiResponse) => void
+ requestWake.mockImplementation(
+ () =>
+ new Promise(resolve => {
+ finishWake = resolve
+ })
+ )
+ const res = makeRes()
+ const attemptUpstream = vi.fn(async () => {})
+ const respondLegacy = vi.fn()
+ const pending = respondWithWakeAndHold(
+ respondOptions(coordinator, res, attemptUpstream, { respondLegacy })
+ )
+
+ res.status(200).json({ success: true })
+ finishWake({ kind: 'active', wakeGeneration: null })
+ await pending
+ expect(attemptUpstream).not.toHaveBeenCalled()
+ expect(respondLegacy).not.toHaveBeenCalled()
+ expect(res.statusCode).toBe(200)
+ expect(coordinator.trackedCoordinationCount()).toBe(0)
+ })
+
it('a readiness probe landing after resolution no-ops loudly instead of re-settling', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake
diff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.ts
index 73c5851c9..a1902149e 100644
--- a/rpc-proxy/src/services/mcpHostRestService.ts
+++ b/rpc-proxy/src/services/mcpHostRestService.ts
@@ -131,7 +131,7 @@ export type HostRuntimeMessageRequest = {
export async function forwardHostMessageToHost(
host: ResolvedServerConnection,
message: HostRuntimeMessageRequest,
- options?: { async?: boolean }
+ options?: { async?: boolean; timeoutMs?: number }
): Promise<Record<string, unknown>> {
const payload = {
...message,
@@ -140,7 +140,15 @@ export async function forwardHostMessageToHost(
const baseUrl = host.url.replace(/\/+$/, '')
const query = options?.async ? '?async=true' : ''
const abortController = new AbortController()
- const timeout = setTimeout(() => abortController.abort(), config.upstreamTimeoutMs)
+ const timeout = setTimeout(() => {
+ if (options?.timeoutMs !== undefined) {
+ const error = new Error('Initial host availability probe timed out')
+ error.name = 'HostAvailabilityProbeTimeoutError'
+ abortController.abort(error)
+ } else {
+ abortController.abort()
+ }
+ }, options?.timeoutMs ?? config.upstreamTimeoutMs)
try {
const response = await fetch(`${baseUrl}/v1/runtime/messages${query}`, {
method: 'POST',
diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.ts
index d6d81c844..59d1ec600 100644
--- a/rpc-proxy/src/services/wakeAndHold.ts
+++ b/rpc-proxy/src/services/wakeAndHold.ts
@@ -39,8 +39,9 @@ const ENTRY_TTL_MARGIN_MS = 5_000
*/
const MAX_TRACKED_WAKE_COORDINATIONS = 1_000
const WAKE_SCOPE = 'host:wake:write' as const
-/** Short upstream retry schedule after a wake reports the host is up. */
-const PROCEED_RETRY_DELAYS_MS = [0, 250, 1_000]
+const ADMISSION_RETRY_DELAYS_MS = [250, 500, 1_000]
+/** Leaves room for one final upstream timeout before Desktop's 60s deadline. */
+export const MAX_REQUEST_HOLD_MS = 48_000
export type WakeHoldOutcome =
/** Host is (or just became) reachable — re-issue the upstream request now. */
@@ -110,6 +111,8 @@ export type WakeHoldParams = {
claims: RpcAccessClaims
/** Raw bearer forwarded to the wake plane when this caller is wake-capable. */
rpcAccessToken: string
+ /** Optional request deadline, shared with upstream admission retries. */
+ deadlineMs?: number
}
/**
@@ -132,7 +135,7 @@ export function wakeCoordinationKey(claims: RpcAccessClaims, hostRef: string): s
* hold budget so a token that cannot survive the wake call is not treated as
* usable for it.
*/
-function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {
+export function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {
const tokenExpMs = claims.exp * 1000
if (tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= now) return false
if (!claims.scopes.includes(WAKE_SCOPE)) return false
@@ -181,9 +184,23 @@ export class WakeAndHoldCoordinator {
lastKnownState: 'unknown',
}
}
- const holdBudgetMs = Math.min(this.deps.maxHoldMs, tokenBudgetMs)
+ const holdBudgetMs = Math.min(
+ this.deps.maxHoldMs,
+ tokenBudgetMs,
+ params.deadlineMs === undefined ? Infinity : Math.max(0, params.deadlineMs - now)
+ )
const deadlineReason =
- holdBudgetMs < this.deps.maxHoldMs ? 'token-expiring' : 'max-hold-exceeded'
+ tokenBudgetMs <= Math.min(this.deps.maxHoldMs, (params.deadlineMs ?? Infinity) - now)
+ ? 'token-expiring'
+ : 'max-hold-exceeded'
+ if (holdBudgetMs <= 0) {
+ return {
+ kind: 'waking',
+ retryAfterMs: this.deps.pollMs,
+ reason: deadlineReason,
+ lastKnownState: 'unknown',
+ }
+ }
const key = wakeCoordinationKey(params.claims, params.hostRef)
const wakeCapable = isWakeCapable(params.claims, params.hostRef, now)
@@ -587,12 +604,14 @@ export type RespondWithWakeAndHoldOptions = {
/** Writes today's error response (502/504) — the pre-wake behavior. */
respondLegacy: (error: unknown) => void
coordinator?: WakeAndHoldCoordinator
+ /** Absolute deadline captured before a route's initial availability probe. */
+ deadlineMs?: number
}
/**
* Drives one held request through the wake flow and always writes a response:
- * - proceed → short upstream retry schedule (draining bounces clear in ~1-2s),
- * then structured host_waking if the pod is still not answering.
+ * - proceed → retry availability failures until upstream accepts or the
+ * single request deadline expires.
* - legacy → today's 502 path (409 not-stateless, wake plane failure, or a
* non-wake-capable caller with no wake authorization to ride).
* - not-found→ 404.
@@ -602,6 +621,8 @@ export async function respondWithWakeAndHold(
options: RespondWithWakeAndHoldOptions
): Promise<void> {
const coordinator = options.coordinator ?? hostWakeCoordinator
+ const deadlineMs =
+ options.deadlineMs ?? Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)
if (options.res.headersSent) {
// A response is already committed for this request: parking it could only
// ever produce a duplicate upstream delivery. Refuse loudly.
@@ -615,12 +636,20 @@ export async function respondWithWakeAndHold(
host: options.host,
claims: options.claims,
rpcAccessToken: options.rpcAccessToken,
+ deadlineMs,
})
+ if (options.res.headersSent) {
+ console.debug(
+ `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${options.hostRef} artifact=hold-outcome`
+ )
+ return
+ }
+
switch (outcome.kind) {
case 'proceed': {
- for (const delayMs of PROCEED_RETRY_DELAYS_MS) {
- if (delayMs > 0) await sleep(delayMs)
+ let availabilityFailures = 0
+ while (Date.now() < deadlineMs) {
if (options.res.headersSent) {
// Resolution latch: the response was committed while this retry
// schedule was pending. Re-forwarding now would duplicate a message
@@ -645,7 +674,15 @@ export async function respondWithWakeAndHold(
)
return
}
- if (isWakeEligibleHostError(error)) continue
+ if (isWakeEligibleHostError(error)) {
+ const remainingMs = deadlineMs - Date.now()
+ const delayMs =
+ ADMISSION_RETRY_DELAYS_MS[
+ Math.min(availabilityFailures++, ADMISSION_RETRY_DELAYS_MS.length - 1)
+ ]
+ if (remainingMs > 0) await sleep(Math.min(delayMs, remainingMs))
+ continue
+ }
// The host answered with a non-availability failure — exactly
// today's behavior for an up-but-erroring host.
console.warn(
@@ -664,7 +701,7 @@ export async function respondWithWakeAndHold(
return
}
console.warn(
- `[RPC_PROXY] wake-hold host still unreachable after wake host=${options.hostRef} lastKnownState=${outcome.lastKnownState}`
+ `[RPC_PROXY] wake-hold admission deadline exceeded host=${options.hostRef} lastKnownState=${outcome.lastKnownState}`
)
respondHostWaking(options.res, {
hostRef: options.hostRef,