Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { Response as ExpressResponse } from 'express'
import { config } from '../../config.js'
import type { ResolvedServerConnection, RpcAccessClaims, RpcScope } from '../../types.js'
import type { HostWakeApiResponse } from '../controlApiRestService.js'
import { UpstreamHostError } from '../mcpHostRestService.js'
import { UpstreamBodyReadError } from '../upstreamBody.js'
import {
WakeAndHoldCoordinator,
type WakeCoordinatorDeps,
describeErrorForLog,
isHostDownNetworkError,
isHostDrainingError,
respondWithWakeAndHold,
} from '../wakeAndHold.js'
const HOST: ResolvedServerConnection = {
name: 'chatllm',
url: 'http://chatllm.mcp-host.svc.cluster.local:8080',
headers: {},
}
const MAX_HOLD_MS = 90_000
const POLL_MS = 2_000
const RETRIGGER_MS = 15_000
function makeCoordinator(overrides?: Partial<WakeCoordinatorDeps>): {
coordinator: WakeAndHoldCoordinator
requestWake: ReturnType<typeof vi.fn>
probeReady: ReturnType<typeof vi.fn>
} {
const requestWake = vi.fn()
const probeReady = vi.fn()
const coordinator = new WakeAndHoldCoordinator({
requestWake,
probeReady,
maxHoldMs: MAX_HOLD_MS,
pollMs: POLL_MS,
retriggerMs: RETRIGGER_MS,
maxTrackedCoordinations: 1_000,
...overrides,
})
return { coordinator, requestWake, probeReady }
}
// Wake-capable by default (§11.2): the branch-matrix below asserts a wake IS
// triggered, which now requires the caller to carry host:wake:write + the
// target hostRef. `tokenExpMs` drives claims.exp so the token-TTL cases still
// exercise the hold budget.
function holdClaims(hostRef: string, tokenExpMs: number, scopes?: RpcScope[]): RpcAccessClaims {
return {
sub: 'user-1',
typ: 'user',
accessScope: 'team',
teamId: 'team-1',
scopes: scopes ?? ['host:wake:write', 'host:message:invoke'],
hostRefs: [hostRef],
jti: 'jti-1',
iat: 1,
exp: Math.floor(tokenExpMs / 1000),
}
}
function holdParams(overrides?: { hostRef?: string; tokenExpMs?: number; scopes?: RpcScope[] }) {
const hostRef = overrides?.hostRef ?? 'chatllm'
const tokenExpMs = overrides?.tokenExpMs ?? Date.now() + 3_600_000
return {
hostRef,
host: HOST,
claims: holdClaims(hostRef, tokenExpMs, overrides?.scopes),
rpcAccessToken: 'rpc-token',
}
}
let warnSpy: ReturnType<typeof vi.spyOn>
let debugSpy: ReturnType<typeof vi.spyOn>
beforeEach(() => {
vi.useFakeTimers()
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => {})
})
afterEach(() => {
expect(vi.getTimerCount()).toBe(0) // no dangling timers, ever
vi.useRealTimers()
warnSpy.mockRestore()
debugSpy.mockRestore()
})
describe('WakeAndHoldCoordinator branch matrix', () => {
it('200 active resolves proceed immediately without any hold loop', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const outcome = await coordinator.hold(holdParams())
expect(outcome).toEqual({ kind: 'proceed', lastKnownState: 'active' })
expect(requestWake).toHaveBeenCalledTimes(1)
expect(requestWake).toHaveBeenCalledWith('chatllm', 'rpc-token')
expect(probeReady).not.toHaveBeenCalled()
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('200 active with wakeGeneration (drain-cancel) also resolves proceed immediately', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: 7 })
const outcome = await coordinator.hold(holdParams())
expect(outcome.kind).toBe('proceed')
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('202 wake-requested holds, polls readiness, and proceeds when the host is up', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 3 })
probeReady.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(POLL_MS) // first probe: not ready
expect(probeReady).toHaveBeenCalledTimes(1)
expect(coordinator.trackedCoordinationCount()).toBe(1)
await vi.advanceTimersByTimeAsync(POLL_MS) // second probe: ready
const outcome = await pending
expect(outcome).toEqual({ kind: 'proceed', lastKnownState: 'wake-requested' })
expect(probeReady).toHaveBeenCalledTimes(2)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('409 not-stateless resolves legacy (today error path)', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'not-stateless' })
const outcome = await coordinator.hold(holdParams())
expect(outcome).toEqual({ kind: 'legacy', reason: 'not-stateless' })
})
it('404 unknown resolves not-found', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'unknown' })
const outcome = await coordinator.hold(holdParams())
expect(outcome).toEqual({ kind: 'not-found' })
})
it('429 rate-limited resolves waking with the control-api Retry-After respected', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'rate-limited', retryAfterSeconds: 30 })
const outcome = await coordinator.hold(holdParams())
expect(outcome).toEqual({
kind: 'waking',
retryAfterMs: 30_000,
reason: 'wake-rate-limited',
lastKnownState: 'unknown',
})
})
it('wake endpoint failure on the initial call resolves legacy, never hangs', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockRejectedValue(new Error('control-api unreachable'))
const outcome = await coordinator.hold(holdParams())
expect(outcome).toEqual({ kind: 'legacy', reason: 'wake-endpoint-unreachable' })
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
})
describe('bounded hold', () => {
it('expires at maxHoldMs with a waking outcome and diagnostics, never exceeding the bound', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
const pending = coordinator.hold(holdParams())
let resolved = false
void pending.then(() => {
resolved = true
})
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS - 1)
expect(resolved).toBe(false)
await vi.advanceTimersByTimeAsync(1)
const outcome = await pending
expect(outcome).toEqual({
kind: 'waking',
retryAfterMs: POLL_MS,
reason: 'max-hold-exceeded',
lastKnownState: 'wake-requested',
})
// Waiter release evicts the whole entry: no residual poll/retrigger/TTL timers.
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
})
describe('token TTL', () => {
it('aborts the hold before the rpc token expires, without any forward', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
// Token expires in 10s: hold budget = 10s - 2s safety margin = 8s.
const pending = coordinator.hold(holdParams({ tokenExpMs: Date.now() + 10_000 }))
await vi.advanceTimersByTimeAsync(8_000)
const outcome = await pending
expect(outcome).toMatchObject({ kind: 'waking', reason: 'token-expiring' })
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('rejects an already-expiring token synchronously without calling the wake endpoint', async () => {
const { coordinator, requestWake } = makeCoordinator()
const outcome = await coordinator.hold(holdParams({ tokenExpMs: Date.now() + 1_000 }))
expect(outcome).toMatchObject({ kind: 'waking', reason: 'token-expiring' })
expect(requestWake).not.toHaveBeenCalled()
})
})
describe('local dedup', () => {
it('N concurrent holds for one host share one wake call and one poll loop, all released on ready', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const pendings = [
coordinator.hold(holdParams()),
coordinator.hold(holdParams()),
coordinator.hold(holdParams()),
]
expect(coordinator.trackedCoordinationCount()).toBe(1)
await vi.advanceTimersByTimeAsync(POLL_MS) // shared probe #1
expect(probeReady).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(POLL_MS) // shared probe #2 → ready
const outcomes = await Promise.all(pendings)
for (const outcome of outcomes) {
expect(outcome.kind).toBe('proceed')
}
expect(requestWake).toHaveBeenCalledTimes(1)
expect(probeReady).toHaveBeenCalledTimes(2)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('evicts the oldest host loudly when the tracked-host cap is exceeded', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator({ maxTrackedCoordinations: 2 })
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
const first = coordinator.hold(holdParams({ hostRef: 'host-a' }))
await vi.advanceTimersByTimeAsync(1)
const second = coordinator.hold(holdParams({ hostRef: 'host-b' }))
await vi.advanceTimersByTimeAsync(1)
expect(coordinator.trackedCoordinationCount()).toBe(2)
const third = coordinator.hold(holdParams({ hostRef: 'host-c' }))
const firstOutcome = await first
expect(firstOutcome).toMatchObject({ kind: 'waking', reason: 'capacity-evicted' })
expect(coordinator.trackedCoordinationCount()).toBe(2)
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('wake-hold capacity exceeded'))
// Release the survivors so afterEach can assert zero dangling timers.
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
await Promise.all([second, third])
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('waiter-deadline expiry evicts an empty entry (TTL semantics), stopping all loops', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
await pending
expect(coordinator.trackedCoordinationCount()).toBe(0)
// Poll/retrigger loops stopped with the entry: advancing time makes no calls.
const probeCalls = probeReady.mock.calls.length
const wakeCalls = requestWake.mock.calls.length
await vi.advanceTimersByTimeAsync(RETRIGGER_MS * 2)
expect(probeReady.mock.calls.length).toBe(probeCalls)
expect(requestWake.mock.calls.length).toBe(wakeCalls)
})
})
describe('kill-switch and poll-source failure while held', () => {
it('re-triggers the wake every retriggerMs while holding', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(RETRIGGER_MS + POLL_MS)
expect(requestWake).toHaveBeenCalledTimes(2) // initial + one retrigger
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
await pending
})
it('skips a retrigger whose wake authorization has expired, keeps holding, and does not settle', async () => {
// Drive the guard through the class's OWN injectable clock (deps.now, read
// via this.now()) rather than mutating global time. That is how the rest of
// this suite controls time, and it keeps the guard's predicate observable
// without a wall-clock side effect.
let clockMs = Date.now()
const { coordinator, requestWake, probeReady } = makeCoordinator({ now: () => clockMs })
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(false)
const tokenExpMs = clockMs + 60_000
const pending = coordinator.hold(holdParams({ tokenExpMs }))
expect(requestWake).toHaveBeenCalledTimes(1) // initial wake only
// Drive the guard's precondition: the retrigger tick lands AFTER the wake
// authorization expired. Advancing the injected clock without advancing the
// timer queue models a blocked event loop / scheduler delay — the entry is
// still unsettled and holding, but its token is now inside the safety margin.
clockMs = tokenExpMs - 1_000
await vi.advanceTimersByTimeAsync(RETRIGGER_MS)
// The doomed POST is NOT spent...
expect(requestWake).toHaveBeenCalledTimes(1)
// ...and the hold survives: the entry is still tracked, not settled, so the
// readiness poll can still release this waiter.
expect(coordinator.trackedCoordinationCount()).toBe(1)
// Drain: let the hold reach its deadline so the pending promise resolves.
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
await pending
})
it('releases held requests via the legacy path when the wake endpoint turns 409 mid-hold', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake
.mockResolvedValueOnce({ kind: 'wake-requested', wakeGeneration: 1 } as HostWakeApiResponse)
.mockResolvedValueOnce({ kind: 'not-stateless' } as HostWakeApiResponse)
probeReady.mockResolvedValue(false)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(RETRIGGER_MS)
const outcome = await pending
expect(outcome).toEqual({ kind: 'legacy', reason: 'not-stateless' })
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('releases held requests when the host becomes ready mid-hold', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(POLL_MS * 2)
const outcome = await pending
expect(outcome.kind).toBe('proceed')
})
it('a persistently erroring poll source still resolves waking at the deadline, never hangs', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockRejectedValue(new Error('status endpoint exploded'))
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
const outcome = await pending
expect(outcome).toMatchObject({ kind: 'waking', reason: 'max-hold-exceeded' })
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('a failing retrigger keeps the hold alive until the deadline instead of crashing it', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake
.mockResolvedValueOnce({ kind: 'wake-requested', wakeGeneration: 1 } as HostWakeApiResponse)
.mockRejectedValue(new Error('control-api blip'))
probeReady.mockResolvedValue(false)
const pending = coordinator.hold(holdParams())
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
const outcome = await pending
expect(outcome).toMatchObject({ kind: 'waking', reason: 'max-hold-exceeded' })
})
})
describe('host error classification', () => {
it('detects the mcp-host DRAINING fence by name/status/body, not class identity', () => {
const draining = Object.assign(new Error('Upstream host returned 503'), {
name: 'UpstreamHostError',
status: 503,
bodySnippet: '{"code":"host_draining"}',
})
expect(isHostDrainingError(draining)).toBe(true)
const plain503 = Object.assign(new Error('Upstream host returned 503'), {
name: 'UpstreamHostError',
status: 503,
bodySnippet: '{"error":"busy"}',
})
expect(isHostDrainingError(plain503)).toBe(false)
})
it('detects network-level host-down failures and excludes AbortError and HTTP answers', () => {
expect(isHostDownNetworkError(new TypeError('fetch failed'))).toBe(true)
const withCause = new Error('request failed')
;(withCause as Error & { cause?: { code: string } }).cause = { code: 'ECONNREFUSED' }
expect(isHostDownNetworkError(withCause)).toBe(true)
const abort = new Error('aborted')
abort.name = 'AbortError'
expect(isHostDownNetworkError(abort)).toBe(false)
const upstream = Object.assign(new Error('Upstream host returned 500'), {
name: 'UpstreamHostError',
status: 500,
bodySnippet: '',
})
expect(isHostDownNetworkError(upstream)).toBe(false)
})
// ADV-HCC-1: the wrapper's message embeds the body failure's text, which can
// itself read like a down host ("read ECONNRESET", "socket hang up"). The
// headers arrived, so it is still never a down-host signal.
it('never classifies a body failure after the headers as a down host', () => {
const bodyFailure = new Error('read ECONNRESET')
// Witness: the same failure raised before fetch() resolved IS a down host.
expect(isHostDownNetworkError(bodyFailure)).toBe(true)
expect(isHostDownNetworkError(new UpstreamBodyReadError(bodyFailure))).toBe(false)
expect(isHostDownNetworkError(new UpstreamBodyReadError(new Error('socket hang up')))).toBe(
false
)
})
})
describe('describeErrorForLog', () => {
const HOST_BODY = '{"error":"boom","detail":"user-content-describe-marker"}'
it('names an UpstreamHostError by status and never by its message', () => {
const error = new UpstreamHostError(500, HOST_BODY)
// Witness: the message really carries the Host body this helper must drop.
expect(error.message).toContain('user-content-describe-marker')
const described = describeErrorForLog(error)
expect(described).toBe('UpstreamHostError status=500')
expect(described).not.toContain('user-content')
})
it('names a network failure by the code on its cause', () => {
const error = new TypeError('fetch failed', { cause: { code: 'ECONNREFUSED' } })
expect(describeErrorForLog(error)).toBe('TypeError code=ECONNREFUSED')
})
it('names a body failure after the headers by the code of the read it wraps', () => {
const read = Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' })
expect(describeErrorForLog(new UpstreamBodyReadError(read))).toBe(
'UpstreamBodyReadError code=ECONNRESET'
)
})
it('keeps only identifier characters of a name and code', () => {
const error = Object.assign(new Error('x'), {
name: 'Bad\nName {"body"}',
code: 'E_CODE\r\n<script>',
})
expect(describeErrorForLog(error)).toBe('BadNamebody code=E_CODEscript')
})
it('reports a thrown non-Error by its type only', () => {
expect(describeErrorForLog('raw upstream text user-content')).toBe('non-error type=string')
})
})
describe('post-resolution re-forward hardening (respondWithWakeAndHold)', () => {
type FakeRes = {
headersSent: boolean
statusCode: number
body: unknown
headers: Record<string, string>
setHeader: (name: string, value: string) => void
status: (code: number) => FakeRes
json: (payload: unknown) => FakeRes
}
function makeRes(): FakeRes {
const res: FakeRes = {
headersSent: false,
statusCode: 0,
body: undefined,
headers: {},
setHeader(name: string, value: string) {
res.headers[name] = value
},
status(code: number) {
res.statusCode = code
return res
},
json(payload: unknown) {
if (res.headersSent) {
throw new Error('Cannot set headers after they are sent to the client')
}
res.headersSent = true
res.body = payload
return res
},
}
return res
}
function upstreamDrainingError(): Error {
return Object.assign(new Error('Upstream host returned 503'), {
name: 'UpstreamHostError',
status: 503,
bodySnippet: '{"code":"host_draining"}',
})
}
function respondOptions(
coordinator: WakeAndHoldCoordinator,
res: FakeRes,
attemptUpstream: (timeoutMs: number, deadlineMs: number) => Promise<void>,
overrides?: {
hostRef?: string
host?: ResolvedServerConnection
respondLegacy?: (error: unknown) => void
retryUntilDeadline?: boolean
}
) {
const hostRef = overrides?.hostRef ?? 'chatllm'
return {
res: res as unknown as ExpressResponse,
hostRef,
host: overrides?.host ?? HOST,
claims: holdClaims(hostRef, Date.now() + 3_600_000),
rpcAccessToken: 'rpc-token',
attemptUpstream,
retryUntilDeadline: overrides?.retryUntilDeadline,
respondLegacy:
overrides?.respondLegacy ??
((_error: unknown) => {
res.status(502).json({ error: 'Upstream host unavailable' })
}),
coordinator,
}
}
it('(a) after a held request resolves 200, advancing every timer window forwards NOTHING further', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const res = makeRes()
const attemptUpstream = vi.fn(async () => {
res.status(200).json({ success: true, taskId: 't-1' })
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
await vi.advanceTimersByTimeAsync(POLL_MS * 2) // probe #2 ready -> proceed -> forward
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(res.statusCode).toBe(200)
expect(coordinator.trackedCoordinationCount()).toBe(0)
const wakeCalls = requestWake.mock.calls.length
const probeCalls = probeReady.mock.calls.length
// Way past every poll, retrigger, entry-TTL, and max-hold window.
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)
expect(attemptUpstream).toHaveBeenCalledTimes(1) // ZERO further upstream forwards
expect(requestWake.mock.calls.length).toBe(wakeCalls)
expect(probeReady.mock.calls.length).toBe(probeCalls)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('(b) after a terminal (non-availability) failure resolves the request, no artifact re-forwards', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
probeReady.mockResolvedValue(true)
const res = makeRes()
const attemptUpstream = vi.fn(async () => {
// Host answered but errored: today's terminal legacy path (never retried).
throw Object.assign(new Error('Upstream host returned 500'), {
name: 'UpstreamHostError',
status: 500,
bodySnippet: '{"error":"boom"}',
})
})
const respondLegacy = vi.fn((_error: unknown) => {
res.status(502).json({ error: 'Upstream host unavailable' })
})
const pending = respondWithWakeAndHold(
respondOptions(coordinator, res, attemptUpstream, { respondLegacy })
)
await vi.advanceTimersByTimeAsync(POLL_MS) // probe ready -> proceed -> failing forward
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(respondLegacy).toHaveBeenCalledTimes(1)
expect(res.statusCode).toBe(502)
expect(coordinator.trackedCoordinationCount()).toBe(0)
const wakeCalls = requestWake.mock.calls.length
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)
expect(attemptUpstream).toHaveBeenCalledTimes(1) // terminal failure stays terminal
expect(requestWake.mock.calls.length).toBe(wakeCalls)
})
it('(c) an unresolved proceed retries availability until admission succeeds', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi
.fn()
.mockRejectedValueOnce(upstreamDrainingError()) // immediate attempt still fenced
.mockRejectedValueOnce(upstreamDrainingError()) // 250ms later still fenced
.mockImplementationOnce(async () => {
res.status(200).json({ success: true, taskId: 't-3' })
})
const pending = respondWithWakeAndHold(
respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })
)
await vi.advanceTimersByTimeAsync(750) // 0ms + 250ms + 750ms attempts
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(3) // pre-resolution retries are the feature
expect(res.statusCode).toBe(200)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('active wake response keeps trying until late upstream admission succeeds within the hold', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
let upstreamAdmits = false
const attemptUpstream = vi.fn(async () => {
if (!upstreamAdmits) throw upstreamDrainingError()
res.status(200).json({ success: true, taskId: 't-late-admission' })
})
const pending = respondWithWakeAndHold(
respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })
)
await vi.advanceTimersByTimeAsync(1_250)
expect(attemptUpstream).toHaveBeenCalledTimes(3)
expect(res.headersSent).toBe(false)
expect(probeReady).not.toHaveBeenCalled()
expect(coordinator.trackedCoordinationCount()).toBe(0)
upstreamAdmits = true
await vi.advanceTimersByTimeAsync(500)
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(4)
expect(res.statusCode).toBe(200)
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)
expect(attemptUpstream).toHaveBeenCalledTimes(4)
})
it('returns host_waking at the absolute deadline without a later admission attempt', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn().mockRejectedValue(upstreamDrainingError())
const pending = respondWithWakeAndHold({
...respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true }),
deadlineMs: Date.now() + 2_500,
})
// Attempts at 0, 250 and 750 ms; at 1750 ms only 750 ms remain, below the
// admission floor, so no fourth attempt is made.
await vi.advanceTimersByTimeAsync(2_500)
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(3)
expect(res.statusCode).toBe(503)
expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
expect(attemptUpstream).toHaveBeenCalledTimes(3)
})
it('passes each admission retry only the remaining hold budget', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
throw upstreamDrainingError()
})
const pending = respondWithWakeAndHold({
...respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true }),
deadlineMs: Date.now() + 1_500,
})
await vi.advanceTimersByTimeAsync(1_500)
await pending
expect(attemptUpstream.mock.calls.map(([timeoutMs]) => timeoutMs)).toEqual([1_500, 1_250])
expect(res.statusCode).toBe(503)
})
it('caps an admission retry timeout at the configured upstream timeout', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
res.status(200).json({ success: true, taskId: 't-capped' })
})
const deadlineMs = Date.now() + config.upstreamTimeoutMs + 5_000
const pending = respondWithWakeAndHold({
...respondOptions(coordinator, res, attemptUpstream),
deadlineMs,
})
await pending
expect(attemptUpstream).toHaveBeenCalledWith(config.upstreamTimeoutMs, deadlineMs)
expect(res.statusCode).toBe(200)
})
// R1-M13: the retry floor sat at 100 ms, below the upstream's admission
// latency, so a retry with a sliver of budget timed out AFTER the upstream
// applied the request (applied-once + 504). Below the floor the caller gets a
// retryable 503 instead.
it('skips an admission retry when less than the 1 s floor remains', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn(async (_timeoutMs: number) => {})
const pending = respondWithWakeAndHold({
...respondOptions(coordinator, res, attemptUpstream),
deadlineMs: Date.now() + 999,
})
await pending
expect(attemptUpstream).not.toHaveBeenCalled()
expect(res.statusCode).toBe(503)
expect(res.body).toMatchObject({ code: 'host_waking' })
})
it('lets a deliberately short upstream timeout retry below the 1 s floor', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
res.status(200).json({ ok: true })
})
const previous = config.upstreamTimeoutMs
config.upstreamTimeoutMs = 300
try {
const pending = respondWithWakeAndHold({
...respondOptions(coordinator, res, attemptUpstream),
deadlineMs: Date.now() + 400,
})
await pending
} finally {
config.upstreamTimeoutMs = previous
}
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(attemptUpstream.mock.calls[0]![0]).toBe(300)
expect(res.statusCode).toBe(200)
})
// R1-M12: without an idempotency key every extra POST is a duplicate side
// effect, so a route that does not opt in gets ONE retry after the wake.
it('re-issues a non-idempotent request at most once, then answers host_waking', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi.fn(async (_timeoutMs: number) => {
throw upstreamDrainingError()
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS)
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(res.statusCode).toBe(503)
expect(res.body).toMatchObject({ code: 'host_waking', hostRef: 'chatllm' })
await vi.advanceTimersByTimeAsync(MAX_HOLD_MS * 3)
expect(attemptUpstream).toHaveBeenCalledTimes(1)
})
it('opting in with retryUntilDeadline keeps re-issuing until the host admits the request', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
const attemptUpstream = vi
.fn()
.mockRejectedValueOnce(upstreamDrainingError())
.mockRejectedValueOnce(upstreamDrainingError())
.mockImplementationOnce(async () => {
res.status(200).json({ ok: true })
})
const pending = respondWithWakeAndHold(
respondOptions(coordinator, res, attemptUpstream, { retryUntilDeadline: true })
)
await vi.advanceTimersByTimeAsync(750)
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(3)
expect(res.statusCode).toBe(200)
})
it('(d) two concurrent holds for different hosts do not cross-cancel each other', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'wake-requested', wakeGeneration: 1 })
const HOST_A: ResolvedServerConnection = {
name: 'host-a',
url: 'http://host-a:8080',
headers: {},
}
const HOST_B: ResolvedServerConnection = {
name: 'host-b',
url: 'http://host-b:8080',
headers: {},
}
let readyB = false
probeReady.mockImplementation(async (host: ResolvedServerConnection) =>
host.name === 'host-a' ? true : readyB
)
const resA = makeRes()
const resB = makeRes()
const attemptA = vi.fn(async () => {
resA.status(200).json({ success: true, taskId: 'a-1' })
})
const attemptB = vi.fn(async () => {
resB.status(200).json({ success: true, taskId: 'b-1' })
})
const pendingA = respondWithWakeAndHold(
respondOptions(coordinator, resA, attemptA, { hostRef: 'host-a', host: HOST_A })
)
const pendingB = respondWithWakeAndHold(
respondOptions(coordinator, resB, attemptB, { hostRef: 'host-b', host: HOST_B })
)
await vi.advanceTimersByTimeAsync(POLL_MS) // A's first probe is ready -> A resolves
await pendingA
expect(attemptA).toHaveBeenCalledTimes(1)
expect(resA.statusCode).toBe(200)
// B is untouched by A's resolution: still held, still polling/retriggering.
expect(attemptB).not.toHaveBeenCalled()
expect(resB.headersSent).toBe(false)
expect(coordinator.trackedCoordinationCount()).toBe(1)
await vi.advanceTimersByTimeAsync(RETRIGGER_MS)
const wakeCallsForB = requestWake.mock.calls.filter(call => call[0] === 'host-b').length
expect(wakeCallsForB).toBe(2) // initial + one retrigger: B's loops survived A's settle
readyB = true
await vi.advanceTimersByTimeAsync(POLL_MS)
await pendingB
expect(attemptB).toHaveBeenCalledTimes(1)
expect(resB.statusCode).toBe(200)
expect(attemptA).toHaveBeenCalledTimes(1) // and B's resolution never re-forwarded A
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('incident shape: a wake-eligible error thrown AFTER the 200 was committed never re-forwards', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const res = makeRes()
// The upstream POST is delivered and the caller's 200 committed, then the
// attempt throws an error the classifier would call "host down/draining".
const attemptUpstream = vi.fn(async () => {
res.status(200).json({ success: true, taskId: 't-dup' })
throw upstreamDrainingError()
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
await vi.advanceTimersByTimeAsync(5_000) // past the whole 0/250/1000ms schedule
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(1) // the duplicate-delivery bug: exactly one forward
expect(res.statusCode).toBe(200)
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('wake-hold post-response failure suppressed')
)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('logs a Host failure raised after the response was committed by status, without its body', async () => {
const { coordinator, requestWake } = makeCoordinator()
requestWake.mockResolvedValue({ kind: 'active', wakeGeneration: null })
const marker = 'user-content-post-response-marker'
const res = makeRes()
const attemptUpstream = vi.fn(async () => {
res.status(200).json({ success: true, taskId: 't-committed' })
throw new UpstreamHostError(500, `{"error":"boom","detail":"${marker}"}`)
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
await vi.advanceTimersByTimeAsync(5_000)
await pending
const lines = [...warnSpy.mock.calls, ...debugSpy.mock.calls].map(args => args.join(' '))
const suppressed = lines.filter(line =>
line.includes('wake-hold post-response failure suppressed (already resolved) host=chatllm')
)
// Witness: the post-response branch ran and logged exactly once.
expect(suppressed).toHaveLength(1)
expect(lines.filter(line => line.includes(marker))).toEqual([])
expect(suppressed[0]).toContain('error=UpstreamHostError status=500')
})
it('refuses to park a request whose response is already committed, loudly and without waking', async () => {
const { coordinator, requestWake } = makeCoordinator()
const res = makeRes()
res.status(200).json({ success: true })
const attemptUpstream = vi.fn(async () => {})
await respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
expect(requestWake).not.toHaveBeenCalled()
expect(attemptUpstream).not.toHaveBeenCalled()
expect(warnSpy).toHaveBeenCalledWith(
expect.stringContaining('wake-hold refused: response already committed')
)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('suppresses a hold outcome after another path commits the response', async () => {
const { coordinator, requestWake } = makeCoordinator()
let finishWake!: (value: HostWakeApiResponse) => void
requestWake.mockImplementation(
() =>
new Promise(resolve => {
finishWake = resolve
})
)
const res = makeRes()
const attemptUpstream = vi.fn(async () => {})
const respondLegacy = vi.fn()
const pending = respondWithWakeAndHold(
respondOptions(coordinator, res, attemptUpstream, { respondLegacy })
)
res.status(200).json({ success: true })
finishWake({ kind: 'active', wakeGeneration: null })
await pending
expect(attemptUpstream).not.toHaveBeenCalled()
expect(respondLegacy).not.toHaveBeenCalled()
expect(res.statusCode).toBe(200)
expect(coordinator.trackedCoordinationCount()).toBe(0)
})
it('a readiness probe landing after resolution no-ops loudly instead of re-settling', async () => {
const { coordinator, requestWake, probeReady } = makeCoordinator()
requestWake
.mockResolvedValueOnce({ kind: 'wake-requested', wakeGeneration: 1 } as HostWakeApiResponse)
.mockResolvedValue({ kind: 'active', wakeGeneration: 1 } as HostWakeApiResponse)
let resolveProbe: ((ready: boolean) => void) | null = null
probeReady.mockImplementation(
() =>
new Promise<boolean>(resolve => {
resolveProbe = resolve
})
)
const res = makeRes()
const attemptUpstream = vi.fn(async () => {
res.status(200).json({ success: true, taskId: 't-late' })
})
const pending = respondWithWakeAndHold(respondOptions(coordinator, res, attemptUpstream))
await vi.advanceTimersByTimeAsync(POLL_MS) // probe #1 starts and hangs
await vi.advanceTimersByTimeAsync(RETRIGGER_MS - POLL_MS) // retrigger: active -> settle proceed
await pending
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(res.statusCode).toBe(200)
expect(coordinator.trackedCoordinationCount()).toBe(0)
// The in-flight probe finally lands AFTER resolution: loud no-op, no re-forward.
expect(resolveProbe).not.toBeNull()
resolveProbe!(true)
await vi.advanceTimersByTimeAsync(0)
expect(attemptUpstream).toHaveBeenCalledTimes(1)
expect(debugSpy).toHaveBeenCalledWith(
expect.stringContaining('late wake-hold artifact ignored (already resolved)')
)
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('artifact=readiness-probe'))
})
})