Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(proxy): admit messages during stateless wake
Alfredo Lopezd09578f1be37
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex fdc83883b..48d93c83f 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -9,7 +9,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.9.0',externalRestApiVersion: '0.1.84',- rpcProxyVersion: '0.1.90',+ rpcProxyVersion: '0.1.91',desktopVersion: '0.9.0',minimumDesktopVersion: '0.1.252',}diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.jsonindex 1e5967806..24d635091 100644--- a/rpc-proxy/package-lock.json+++ b/rpc-proxy/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-rpc-proxy",- "version": "0.1.90",+ "version": "0.1.91","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-rpc-proxy",- "version": "0.1.90",+ "version": "0.1.91","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/rpc-proxy/package.json b/rpc-proxy/package.jsonindex cf15847a3..6b775fd3f 100644--- a/rpc-proxy/package.json+++ b/rpc-proxy/package.json@@ -1,6 +1,6 @@{"name": "clerum-rpc-proxy",- "version": "0.1.90",+ "version": "0.1.91","description": "External-facing RPC proxy for tenant-scoped MCP access","main": "dist/main.js","engines": {diff --git a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts b/rpc-proxy/src/__tests__/wake-and-hold-route.test.tsindex bf37f6bf4..0fd4d72af 100644--- a/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts+++ b/rpc-proxy/src/__tests__/wake-and-hold-route.test.ts@@ -1,4 +1,4 @@-import { beforeEach, describe, expect, it, vi } from 'vitest'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import request from 'supertest'import { createRpcRouter } from '../routes/rpc.js'@@ -96,6 +96,10 @@ beforeEach(() => {serviceMock.resolveHostConnectionForUser.mockResolvedValue({ ...HOST_CONNECTION })})+afterEach(() => {+ vi.useRealTimers()+})+describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {it('host-down network error triggers a wake; 200 active leads to one immediate upstream retry', async () => {serviceMock.forwardHostMessageToHost@@ -114,6 +118,51 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)})+ it('starts wake after a short availability probe and admits after the Host becomes ready', async () => {+ let firstForwardStarted!: () => void+ const firstForward = new Promise<void>(resolve => {+ firstForwardStarted = resolve+ })+ let upstreamAdmits = false+ const availabilityTimeout = new Error('Initial host availability probe timed out')+ availabilityTimeout.name = 'HostAvailabilityProbeTimeoutError'+ serviceMock.forwardHostMessageToHost+ .mockImplementationOnce(+ (_host: unknown, _body: unknown, options: { timeoutMs?: number }) =>+ new Promise((_resolve, reject) => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ expect(options.timeoutMs).toBe(750)+ firstForwardStarted()+ setTimeout(() => reject(availabilityTimeout), options.timeoutMs)+ })+ )+ .mockImplementation(async () => {+ if (!upstreamAdmits) throw hostDownError()+ return { success: true, taskId: 't-delayed' }+ })+ controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({+ kind: 'active',+ wakeGeneration: null,+ })++ const responsePromise = postMessage(makeApp())+ .expect(200)+ .then(response => response)+ await firstForward+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(750)+ expect(controlApiMock.requestHostWakeFromControlApi).toHaveBeenCalledTimes(1)+ expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(1_500)+ upstreamAdmits = true+ await vi.advanceTimersByTimeAsync(250)+ const response = await responsePromise+ expect(response.body).toEqual({ success: true, taskId: 't-delayed' })+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][1]).toBe(+ serviceMock.forwardHostMessageToHost.mock.calls.at(-1)![1]+ )+ })+it('the wake retry re-forwards the SAME stable messageId as the first forward (idempotency identity)', async () => {// P1-1: without a stable delivery id the wake-and-hold retry re-forwards// the same turn under a fresh uuid at mcp-host and executes it twice. The@@ -277,17 +326,26 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {})it('host still unreachable after a wake reports active resolves to host_waking, never a hang', async () => {+ let wakeStarted!: () => void+ const wakeCalled = new Promise<void>(resolve => {+ wakeStarted = resolve+ })serviceMock.forwardHostMessageToHost.mockRejectedValue(hostDownError())- controlApiMock.requestHostWakeFromControlApi.mockResolvedValue({- kind: 'active',- wakeGeneration: null,+ controlApiMock.requestHostWakeFromControlApi.mockImplementation(async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })+ wakeStarted()+ return { kind: 'active', wakeGeneration: null }})- const response = await postMessage(makeApp()).expect(503)+ const responsePromise = postMessage(makeApp())+ .expect(503)+ .then(response => response)+ await wakeCalled+ await vi.advanceTimersByTimeAsync(48_000)+ const response = await responsePromiseexpect(response.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })- // Trigger + the full short retry schedule were attempted.- expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(4)+ expect(serviceMock.forwardHostMessageToHost.mock.calls.length).toBeGreaterThan(4)})it('a non-availability upstream failure (mcp-host 500) never triggers a wake', async () => {@@ -311,6 +369,22 @@ describe('POST /rpc/hosts/:hostRef/messages wake-and-hold triggers', () => {expect(response.body).toEqual({ error: 'Gateway Timeout' })expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()})++ it('a token without wake scope keeps the original full upstream timeout', async () => {+ authTokenMock.verifyRpcToken.mockReturnValue({+ ...VALID_CLAIMS,+ scopes: ['host:message:invoke'],+ })+ const abort = new Error('aborted')+ abort.name = 'AbortError'+ serviceMock.forwardHostMessageToHost.mockRejectedValue(abort)++ await postMessage(makeApp()).expect(504)++ expect(serviceMock.forwardHostMessageToHost).toHaveBeenCalledTimes(1)+ expect(serviceMock.forwardHostMessageToHost.mock.calls[0][2]).toEqual({ async: false })+ expect(controlApiMock.requestHostWakeFromControlApi).not.toHaveBeenCalled()+ })})describe('task result and cancel host-down paths', () => {diff --git a/rpc-proxy/src/routes/rpc.ts b/rpc-proxy/src/routes/rpc.tsindex b7027712b..829721aec 100644--- a/rpc-proxy/src/routes/rpc.ts+++ b/rpc-proxy/src/routes/rpc.ts@@ -32,7 +32,9 @@ import {validateRpcRequest,} from '../services/mcpProxyService.js'import {+ MAX_REQUEST_HOLD_MS,isUpstreamTimeoutError,+ isWakeCapable,isWakeEligibleHostError,respondWithWakeAndHold,} from '../services/wakeAndHold.js'@@ -45,6 +47,8 @@ const RFC1123_RE = /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/// own copy (the two services share no package, so each names its own).const SESSIONS_LIMIT_CAP = 100const MESSAGES_LIMIT_CAP = 200+/** A suspended pod may spend seconds failing DNS/connect before the wake starts. */+const MESSAGE_AVAILABILITY_PROBE_MS = 750function isSafeUpstreamPathSegment(value: string): boolean {return (@@ -403,6 +407,7 @@ export function createRpcRouter(): Router {: {}),}+ const wakeDeadlineMs = Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)const host = await resolveHostConnectionForUser(auth.sub, hostRef, rpcAccessToken, {teamId: auth.teamId,...(traceContext.sessionId@@ -437,9 +442,11 @@ export function createRpcRouter(): Router {`[RPC_PROXY] user=${auth.sub} host=${hostRef} method=host-message-rest${isAsync ? ' (async)' : ''} attachments=${attachmentCount}`)let upstreamResponse: Record<string, unknown> | null = null+ const wakeCapable = isWakeCapable(auth, hostRef, Date.now())try {upstreamResponse = await forwardHostMessageToHost(host, forwardedBody, {async: isAsync,+ ...(wakeCapable ? { timeoutMs: MESSAGE_AVAILABILITY_PROBE_MS } : {}),})} catch (error) {console.warn(@@ -447,7 +454,12 @@ export function createRpcRouter(): Router {error instanceof Error ? error.message : String(error)}`)- if (isWakeEligibleHostError(error)) {+ if (+ isWakeEligibleHostError(error) ||+ (wakeCapable &&+ error instanceof Error &&+ error.name === 'HostAvailabilityProbeTimeoutError')+ ) {// Stateless wake-and-hold: a down or draining host triggers a// control-api wake (and possibly a bounded hold) instead of the// generic 502. Non-stateless hosts fall back to the legacy path.@@ -457,6 +469,7 @@ export function createRpcRouter(): Router {host,claims: auth,rpcAccessToken,+ deadlineMs: wakeDeadlineMs,attemptUpstream: async () => {const retried = await forwardHostMessageToHost(host, forwardedBody, {async: isAsync,diff --git a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.tsindex 795f85994..5f6ac615d 100644--- a/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts+++ b/rpc-proxy/src/services/__tests__/mcpHostRestService.test.ts@@ -1,5 +1,36 @@-import { describe, expect, it } from 'vitest'-import { __test__normalizeHostStatusPayload as normalize } from '../mcpHostRestService.js'+import { describe, expect, it, vi } from 'vitest'+import {+ forwardHostMessageToHost,+ __test__normalizeHostStatusPayload as normalize,+} from '../mcpHostRestService.js'++describe('host message availability probe', () => {+ it('aborts at the requested short timeout with a distinct wake-eligible reason', async () => {+ vi.useFakeTimers()+ const fetchMock = vi.spyOn(globalThis, 'fetch').mockImplementation(+ (_input, init) =>+ new Promise((_resolve, reject) => {+ init?.signal?.addEventListener('abort', () => reject(init.signal?.reason))+ })+ )+ try {+ const pending = forwardHostMessageToHost(+ { name: 'chatllm', url: 'http://chatllm:8080', headers: {} },+ { content: 'hello', hostRef: 'chatllm' },+ { timeoutMs: 750 }+ )+ const rejection = expect(pending).rejects.toMatchObject({+ name: 'HostAvailabilityProbeTimeoutError',+ })+ await vi.advanceTimersByTimeAsync(750)+ await rejection+ expect(fetchMock).toHaveBeenCalledTimes(1)+ } finally {+ fetchMock.mockRestore()+ vi.useRealTimers()+ }+ })+})const baseUpstream = {agent: {diff --git a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts b/rpc-proxy/src/services/__tests__/wakeAndHold.test.tsindex 0758e8c7f..da48afd64 100644--- a/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts+++ b/rpc-proxy/src/services/__tests__/wakeAndHold.test.ts@@ -559,7 +559,7 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>expect(requestWake.mock.calls.length).toBe(wakeCalls)})- it('(c) an unresolved proceed still walks the short retry schedule exactly as today', async () => {+ it('(c) an unresolved proceed retries availability until admission succeeds', async () => {const { coordinator, requestWake } = makeCoordinator()requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })@@ -573,14 +573,14 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>})const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))- await vi.advanceTimersByTimeAsync(1_250) // 0ms + 250ms + 1000ms schedule+ await vi.advanceTimersByTimeAsync(750) // 0ms + 250ms + 750ms attemptsawait pendingexpect(attemptUpstream).toHaveBeenCalledTimes(3) // pre-resolution retries are the featureexpect(res.statusCode).toBe(200)expect(coordinator.trackedCoordinationCount()).toBe(0)})- it('active wake response currently returns host_waking when upstream admission becomes possible after the bounded retry schedule', async () => {+ it('active wake response keeps trying until late upstream admission succeeds within the hold', async () => {const { coordinator, requestWake, probeReady } = makeCoordinator()requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })@@ -592,22 +592,38 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>})const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))- await vi.advanceTimersByTimeAsync(1_250) // exhaust 0ms + 250ms + 1000ms attempts- await pending+ await vi.advanceTimersByTimeAsync(1_250)expect(attemptUpstream).toHaveBeenCalledTimes(3)- expect(res.statusCode).toBe(503)- expect(res.body).toMatchObject({- code: 'host_waking',- hostRef: 'chatllm',- lastKnownState: 'active',- })+ expect(res.headersSent).toBe(false)expect(probeReady).not.toHaveBeenCalled()expect(coordinator.trackedCoordinationCount()).toBe(0)upstreamAdmits = true+ await vi.advanceTimersByTimeAsync(500)+ await pending+ expect(attemptUpstream).toHaveBeenCalledTimes(4)+ expect(res.statusCode).toBe(200)await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)- expect(attemptUpstream).toHaveBeenCalledTimes(3) // admission later does not trigger a retry+ expect(attemptUpstream).toHaveBeenCalledTimes(4)+ })++ it('returns host_waking at the absolute deadline without a later admission attempt', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })+ const res = makeRes()+ const attemptUpstream = vi.fn().mockRejectedValue(upstreamDrainingError())+ const pending = respondWithWakeAndHold({+ ...respondOptions(coordinator, res, attemptUpstream),+ deadlineMs: Date.now() + 1_000,+ })++ await vi.advanceTimersByTimeAsync(1_000)+ await pending+ expect(attemptUpstream).toHaveBeenCalledTimes(3)expect(res.statusCode).toBe(503)+ expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })+ await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)+ expect(attemptUpstream).toHaveBeenCalledTimes(3)})it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {@@ -704,6 +720,31 @@ describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () =>expect(coordinator.trackedCoordinationCount()).toBe(0)})+ it('suppresses a hold outcome after another path commits the response', async () => {+ const { coordinator, requestWake } = makeCoordinator()+ let finishWake!: (value: HostWakeApiResponse) => void+ requestWake.mockImplementation(+ () =>+ new Promise(resolve => {+ finishWake = resolve+ })+ )+ const res = makeRes()+ const attemptUpstream = vi.fn(async () => {})+ const respondLegacy = vi.fn()+ const pending = respondWithWakeAndHold(+ respondOptions(coordinator, res, attemptUpstream, { respondLegacy })+ )++ res.status(200).json({ success: true })+ finishWake({ kind: 'active', wakeGeneration: null })+ await pending+ expect(attemptUpstream).not.toHaveBeenCalled()+ expect(respondLegacy).not.toHaveBeenCalled()+ expect(res.statusCode).toBe(200)+ expect(coordinator.trackedCoordinationCount()).toBe(0)+ })+it('a readiness probe landing after resolution no-ops loudly instead of re-settling', async () => {const { coordinator, requestWake, probeReady } = makeCoordinator()requestWakediff --git a/rpc-proxy/src/services/mcpHostRestService.ts b/rpc-proxy/src/services/mcpHostRestService.tsindex 73c5851c9..a1902149e 100644--- a/rpc-proxy/src/services/mcpHostRestService.ts+++ b/rpc-proxy/src/services/mcpHostRestService.ts@@ -131,7 +131,7 @@ export type HostRuntimeMessageRequest = {export async function forwardHostMessageToHost(host: ResolvedServerConnection,message: HostRuntimeMessageRequest,- options?: { async?: boolean }+ options?: { async?: boolean; timeoutMs?: number }): Promise<Record<string, unknown>> {const payload = {...message,@@ -140,7 +140,15 @@ export async function forwardHostMessageToHost(const baseUrl = host.url.replace(/\/+$/, '')const query = options?.async ? '?async=true' : ''const abortController = new AbortController()- const timeout = setTimeout(() => abortController.abort(), config.upstreamTimeoutMs)+ const timeout = setTimeout(() => {+ if (options?.timeoutMs !== undefined) {+ const error = new Error('Initial host availability probe timed out')+ error.name = 'HostAvailabilityProbeTimeoutError'+ abortController.abort(error)+ } else {+ abortController.abort()+ }+ }, options?.timeoutMs ?? config.upstreamTimeoutMs)try {const response = await fetch(`${baseUrl}/v1/runtime/messages${query}`, {method: 'POST',diff --git a/rpc-proxy/src/services/wakeAndHold.ts b/rpc-proxy/src/services/wakeAndHold.tsindex d6d81c844..59d1ec600 100644--- a/rpc-proxy/src/services/wakeAndHold.ts+++ b/rpc-proxy/src/services/wakeAndHold.ts@@ -39,8 +39,9 @@ const ENTRY_TTL_MARGIN_MS = 5_000*/const MAX_TRACKED_WAKE_COORDINATIONS = 1_000const WAKE_SCOPE = 'host:wake:write' as const-/** Short upstream retry schedule after a wake reports the host is up. */-const PROCEED_RETRY_DELAYS_MS = [0, 250, 1_000]+const ADMISSION_RETRY_DELAYS_MS = [250, 500, 1_000]+/** Leaves room for one final upstream timeout before Desktop's 60s deadline. */+export const MAX_REQUEST_HOLD_MS = 48_000export type WakeHoldOutcome =/** Host is (or just became) reachable — re-issue the upstream request now. */@@ -110,6 +111,8 @@ export type WakeHoldParams = {claims: RpcAccessClaims/** Raw bearer forwarded to the wake plane when this caller is wake-capable. */rpcAccessToken: string+ /** Optional request deadline, shared with upstream admission retries. */+ deadlineMs?: number}/**@@ -132,7 +135,7 @@ export function wakeCoordinationKey(claims: RpcAccessClaims, hostRef: string): s* hold budget so a token that cannot survive the wake call is not treated as* usable for it.*/-function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {+export function isWakeCapable(claims: RpcAccessClaims, hostRef: string, now: number): boolean {const tokenExpMs = claims.exp * 1000if (tokenExpMs - TOKEN_EXP_SAFETY_MARGIN_MS <= now) return falseif (!claims.scopes.includes(WAKE_SCOPE)) return false@@ -181,9 +184,23 @@ export class WakeAndHoldCoordinator {lastKnownState: 'unknown',}}- const holdBudgetMs = Math.min(this.deps.maxHoldMs, tokenBudgetMs)+ const holdBudgetMs = Math.min(+ this.deps.maxHoldMs,+ tokenBudgetMs,+ params.deadlineMs === undefined ? Infinity : Math.max(0, params.deadlineMs - now)+ )const deadlineReason =- holdBudgetMs < this.deps.maxHoldMs ? 'token-expiring' : 'max-hold-exceeded'+ tokenBudgetMs <= Math.min(this.deps.maxHoldMs, (params.deadlineMs ?? Infinity) - now)+ ? 'token-expiring'+ : 'max-hold-exceeded'+ if (holdBudgetMs <= 0) {+ return {+ kind: 'waking',+ retryAfterMs: this.deps.pollMs,+ reason: deadlineReason,+ lastKnownState: 'unknown',+ }+ }const key = wakeCoordinationKey(params.claims, params.hostRef)const wakeCapable = isWakeCapable(params.claims, params.hostRef, now)@@ -587,12 +604,14 @@ export type RespondWithWakeAndHoldOptions = {/** Writes today's error response (502/504) — the pre-wake behavior. */respondLegacy: (error: unknown) => voidcoordinator?: WakeAndHoldCoordinator+ /** Absolute deadline captured before a route's initial availability probe. */+ deadlineMs?: number}/*** Drives one held request through the wake flow and always writes a response:- * - proceed → short upstream retry schedule (draining bounces clear in ~1-2s),- * then structured host_waking if the pod is still not answering.+ * - proceed → retry availability failures until upstream accepts or the+ * single request deadline expires.* - legacy → today's 502 path (409 not-stateless, wake plane failure, or a* non-wake-capable caller with no wake authorization to ride).* - not-found→ 404.@@ -602,6 +621,8 @@ export async function respondWithWakeAndHold(options: RespondWithWakeAndHoldOptions): Promise<void> {const coordinator = options.coordinator ?? hostWakeCoordinator+ const deadlineMs =+ options.deadlineMs ?? Date.now() + Math.min(config.wakeMaxHoldMs, MAX_REQUEST_HOLD_MS)if (options.res.headersSent) {// A response is already committed for this request: parking it could only// ever produce a duplicate upstream delivery. Refuse loudly.@@ -615,12 +636,20 @@ export async function respondWithWakeAndHold(host: options.host,claims: options.claims,rpcAccessToken: options.rpcAccessToken,+ deadlineMs,})+ if (options.res.headersSent) {+ console.debug(+ `[RPC_PROXY] late wake-hold artifact ignored (already resolved) host=${options.hostRef} artifact=hold-outcome`+ )+ return+ }+switch (outcome.kind) {case 'proceed': {- for (const delayMs of PROCEED_RETRY_DELAYS_MS) {- if (delayMs > 0) await sleep(delayMs)+ let availabilityFailures = 0+ while (Date.now() < deadlineMs) {if (options.res.headersSent) {// Resolution latch: the response was committed while this retry// schedule was pending. Re-forwarding now would duplicate a message@@ -645,7 +674,15 @@ export async function respondWithWakeAndHold()return}- if (isWakeEligibleHostError(error)) continue+ if (isWakeEligibleHostError(error)) {+ const remainingMs = deadlineMs - Date.now()+ const delayMs =+ ADMISSION_RETRY_DELAYS_MS[+ Math.min(availabilityFailures++, ADMISSION_RETRY_DELAYS_MS.length - 1)+ ]+ if (remainingMs > 0) await sleep(Math.min(delayMs, remainingMs))+ continue+ }// The host answered with a non-availability failure — exactly// today's behavior for an up-but-erroring host.console.warn(@@ -664,7 +701,7 @@ export async function respondWithWakeAndHold(return}console.warn(- `[RPC_PROXY] wake-hold host still unreachable after wake host=${options.hostRef} lastKnownState=${outcome.lastKnownState}`+ `[RPC_PROXY] wake-hold admission deadline exceeded host=${options.hostRef} lastKnownState=${outcome.lastKnownState}`)respondHostWaking(options.res, {hostRef: options.hostRef,