Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(llm-proxy): keep silent attempts open with SSE keepalive comments

Alfredo Lopez5aaf89a61cde
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index aed7bc498..5addba8dc 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -83,6 +83,12 @@ export type StreamCodexCompletionInput = {
hostRef: string
operation: 'completion_stream'
}) => Promise<RedeemAttemptSuccess>
+ /**
+ * Called once, after the redeem succeeded and its served model and deadline
+ * were accepted, and before the upstream fetch. The server starts its SSE
+ * heartbeat here, so a denied redeem still answers with an HTTP status.
+ */
+ onRedeemed?: () => void
finalize: (input: {
attemptReceipt: string
receipt: {
@@ -139,6 +145,7 @@ export async function streamCodexCompletion(
redeemed.transport.maxStreamDurationMs
)
const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
+ input.onRedeemed?.()
const accessToken = redeemed.accessToken
let outcome: StreamCodexCompletionResult['outcome'] = 'unknown'
diff --git a/codex-llm-proxy/src/config.ts b/codex-llm-proxy/src/config.ts
index 610850fb3..2084bd5f8 100644
--- a/codex-llm-proxy/src/config.ts
+++ b/codex-llm-proxy/src/config.ts
@@ -1,4 +1,5 @@
import { STREAM_LIMITS } from './requestLimits.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
export type CodexLlmProxyConfig = {
runtimePort: number
@@ -9,6 +10,8 @@ export type CodexLlmProxyConfig = {
maxDeadlineMs: number
/** Lowers STREAM_LIMITS.upstreamIdleTimeoutMs; the transport never raises it. */
upstreamIdleTimeoutMs: number
+ /** Interval between SSE keepalive comments once the redeem succeeded. */
+ heartbeatIntervalMs: number
jwtIssuer: string
jwtPublicKey: string
executionEnabled: boolean
@@ -84,6 +87,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyC
env.CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,
STREAM_LIMITS.upstreamIdleTimeoutMs
),
+ heartbeatIntervalMs: requiredPositiveInt(
+ 'CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
+ env.CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
+ DEFAULT_HEARTBEAT_INTERVAL_MS
+ ),
jwtIssuer: env.CODEX_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',
jwtPublicKey: requiredPem('CODEX_LLM_PROXY_JWT_PUBLIC_KEY', env.CODEX_LLM_PROXY_JWT_PUBLIC_KEY),
executionEnabled: env.CODEX_LLM_PROXY_EXECUTION_ENABLED === 'true',
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 01047c747..d0891352c 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -23,6 +23,7 @@ import {
type OriginPolicyOptions,
} from './originPolicy.js'
import { RequestLimitError, streamGate } from './requestLimits.js'
+import { startSseHeartbeat } from './sseHeartbeat.js'
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])
@@ -173,6 +174,10 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
const attemptStarted = Date.now()
let toolCalls = 0
let textChunks = 0
+ let heartbeats = 0
+ // Set once the redeem succeeded. Every exit path below calls it before
+ // it ends the response: a tick after `res.end()` would write after end.
+ let stopHeartbeat: (() => void) | undefined
try {
release = await streamGate.acquire(abort.signal)
res.status(200)
@@ -195,6 +200,12 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
},
signal: abort.signal,
redeem: input => client.redeem(input),
+ onRedeemed: () => {
+ stopHeartbeat = startSseHeartbeat(res, abort.signal, config.heartbeatIntervalMs, () => {
+ heartbeats += 1
+ })
+ abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
+ },
finalize: input => client.finalize(input),
fetchFn,
lookup,
@@ -204,6 +215,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
},
})
+ stopHeartbeat?.()
res.write(
`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
)
@@ -216,6 +228,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
deliveredAs: 'sse_done',
toolCalls,
textChunks,
+ heartbeats,
durationMs: Date.now() - attemptStarted,
...(result.usage ? { usage: result.usage } : {}),
}
@@ -223,6 +236,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
else logger.warn(finished, 'codex attempt finished')
res.end()
} catch (err) {
+ stopHeartbeat?.()
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
metrics.observeAttemptFailure(failureLabel(mapped.code))
@@ -243,6 +257,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
toolCalls,
textChunks,
+ heartbeats,
durationMs: Date.now() - attemptStarted,
},
'codex attempt finished'
@@ -258,6 +273,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
res.setHeader('content-type', 'application/json; charset=utf-8')
res.status(mapped.status).json({ error: mapped.code })
} finally {
+ stopHeartbeat?.()
release?.()
}
})()
diff --git a/codex-llm-proxy/src/sseHeartbeat.ts b/codex-llm-proxy/src/sseHeartbeat.ts
new file mode 100644
index 000000000..06029f4c7
--- /dev/null
+++ b/codex-llm-proxy/src/sseHeartbeat.ts
@@ -0,0 +1,43 @@
+/** The part of a server response the heartbeat touches. */
+export type HeartbeatSink = {
+ write(chunk: string): boolean
+ readonly writableEnded: boolean
+ readonly destroyed: boolean
+}
+
+const KEEPALIVE = ': keepalive\n\n'
+
+/**
+ * Well under the Host's 300 s undici headers/body timeouts, so a silent
+ * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one
+ * interval stays below 300 s.
+ */
+export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
+
+/**
+ * Writes an SSE comment every `intervalMs` so the Host's HTTP client, whose
+ * headers and body timeouts are 300 s, keeps the attempt open while the
+ * upstream is silent (reasoning, or tool calls that the proxy buffers until
+ * the stream completes). SSE readers ignore comment lines.
+ *
+ * The first comment goes out at +intervalMs. A tick never waits on `drain`
+ * and never adds a listener: a comment is 13 bytes, and a slow client is
+ * already bounded by the frame writer's backpressure. The caller must call
+ * the returned stop function before ending the response.
+ */
+export function startSseHeartbeat(
+ res: HeartbeatSink,
+ signal: AbortSignal,
+ intervalMs: number,
+ onBeat: () => void
+): () => void {
+ if (!Number.isInteger(intervalMs) || intervalMs <= 0) {
+ throw new Error('heartbeat interval must be a positive integer')
+ }
+ const timer = setInterval(() => {
+ if (res.writableEnded || res.destroyed || signal.aborted) return
+ res.write(KEEPALIVE)
+ onBeat()
+ }, intervalMs)
+ return () => clearInterval(timer)
+}
diff --git a/codex-llm-proxy/test/codexTransport.conformance.test.ts b/codex-llm-proxy/test/codexTransport.conformance.test.ts
index 2b064a39e..3f0118758 100644
--- a/codex-llm-proxy/test/codexTransport.conformance.test.ts
+++ b/codex-llm-proxy/test/codexTransport.conformance.test.ts
@@ -678,6 +678,68 @@ describe('streamCodexCompletion', () => {
).rejects.toMatchObject({ code: 'origin_denied' })
})
+ it('calls onRedeemed once, after a matching redeem and before the upstream fetch', async () => {
+ const ticket = {
+ jti: 'jti-redeemed',
+ hostRef: 'research-host',
+ model: REQUEST.model,
+ requestHash: REQUEST_HASH,
+ providerAttemptId: 'att-redeemed',
+ }
+ const finalize = vi.fn(async () => ({
+ providerAttemptId: 'att-redeemed',
+ outcome: 'success' as const,
+ duplicate: false,
+ }))
+ const order: string[] = []
+ await streamCodexCompletion({
+ executionTicket: 'ticket-redeemed',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket,
+ redeem: async () => {
+ order.push('redeem')
+ return redeemSuccess()
+ },
+ onRedeemed: () => order.push('onRedeemed'),
+ finalize,
+ fetchFn: vi.fn(async () => {
+ order.push('fetch')
+ return sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
+ }),
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ })
+ expect(order).toEqual(['redeem', 'onRedeemed', 'fetch'])
+
+ // A denied redeem and a served-model mismatch never start the heartbeat.
+ const denied = vi.fn(async (): Promise<RedeemAttemptSuccess> => {
+ throw new Error('no_grant')
+ })
+ const mismatched = vi.fn(async () =>
+ redeemSuccess({ transport: { ...redeemSuccess().transport, servedModel: 'other' } })
+ )
+ const onRedeemed = vi.fn()
+ for (const redeem of [denied, mismatched]) {
+ await expect(
+ streamCodexCompletion({
+ executionTicket: 'ticket-redeemed',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket,
+ redeem,
+ onRedeemed,
+ finalize,
+ fetchFn: vi.fn(),
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ })
+ ).rejects.toThrow()
+ }
+ // Witness: both redeems ran, so the path that could call onRedeemed was entered.
+ expect(denied).toHaveBeenCalledTimes(1)
+ expect(mismatched).toHaveBeenCalledTimes(1)
+ expect(onRedeemed).not.toHaveBeenCalled()
+ })
+
it('follows one frozen same-origin redirect then streams', async () => {
const fetchFn = vi.fn(async () => {
if (fetchFn.mock.calls.length === 1) {
diff --git a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
index 868bf049f..13529acd4 100644
--- a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
+++ b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
@@ -83,6 +83,7 @@ function config(): CodexLlmProxyConfig {
maxStreamDurationMs: 30_000,
maxDeadlineMs: 30_000,
upstreamIdleTimeoutMs: 300_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index 4db724354..b0f930b8c 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -34,6 +34,7 @@ function config(overrides: Partial<CodexLlmProxyConfig> = {}): CodexLlmProxyConf
maxStreamDurationMs: 300_000,
maxDeadlineMs: 300_000,
upstreamIdleTimeoutMs: 300_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
@@ -317,6 +318,23 @@ describe('codex-llm-proxy security surface', () => {
}).upstreamIdleTimeoutMs
).toBe(45_000)
})
+
+ it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
+ const env = {
+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ CODEX_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ CODEX_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ }
+ expect(loadConfig(env).heartbeatIntervalMs).toBe(15_000)
+ expect(
+ loadConfig({ ...env, CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '5000' }).heartbeatIntervalMs
+ ).toBe(5_000)
+ for (const raw of ['0', '-1']) {
+ expect(() => loadConfig({ ...env, CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
+ /CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be a finite integer greater than zero/
+ )
+ }
+ })
})
describe('codex-llm-proxy execution kill switch', () => {
@@ -628,6 +646,59 @@ describe('codex-llm-proxy attempt telemetry', () => {
return line ? Number(line.split(' ').pop()) : 0
}
+ function keepaliveCount(text: string): number {
+ return text.split(': keepalive\n\n').length - 1
+ }
+
+ // Stays silent for `silentMs` after the fetch resolves, then streams one text
+ // delta and completes, like a model that reasons before its first token.
+ function silentThenCompletingUpstream(silentMs: number): typeof fetch {
+ return (async () =>
+ new Response(
+ new ReadableStream<Uint8Array>({
+ async start(controller) {
+ await new Promise(resolve => setTimeout(resolve, silentMs))
+ controller.enqueue(
+ new TextEncoder().encode(
+ 'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
+ 'data: {"type":"response.completed"}\n\n'
+ )
+ )
+ controller.close()
+ },
+ }),
+ { status: 200, headers: { 'content-type': 'text/event-stream' } }
+ )) as typeof fetch
+ }
+
+ // Answers `status` with no SSE body after `delayMs`.
+ function slowFailingUpstream(delayMs: number, status: number): typeof fetch {
+ return (async () => {
+ await new Promise(resolve => setTimeout(resolve, delayMs))
+ return new Response('upstream failed', { status })
+ }) as typeof fetch
+ }
+
+ // Denies the redeem after `delayMs`, which is longer than the heartbeat
+ // interval the caller configures.
+ function slowDenyingClient(code: string, delayMs: number): {
+ client: ControlApiClient
+ redeemCalls: () => number
+ } {
+ let calls = 0
+ const client = {
+ async redeem() {
+ calls += 1
+ await new Promise(resolve => setTimeout(resolve, delayMs))
+ throw new ControlApiClientError(code, 'control API request denied')
+ },
+ async finalize() {
+ throw new Error('finalize must not run after a denied redeem')
+ },
+ } as unknown as ControlApiClient
+ return { client, redeemCalls: () => calls }
+ }
+
async function run(
providerAttemptId: string,
textDeltas: number,
@@ -638,13 +709,14 @@ describe('codex-llm-proxy attempt telemetry', () => {
fetchFn?: typeof fetch
maxStreamDurationMs?: number
configOverrides?: Partial<CodexLlmProxyConfig>
+ controlApiClient?: ControlApiClient
} = {}
) {
const info = vi.spyOn(logger, 'info')
const warn = vi.spyOn(logger, 'warn')
const { client, receipts } = grantingClient(options.maxStreamDurationMs)
const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {
- controlApiClient: deniedCode ? denyingClient(deniedCode) : client,
+ controlApiClient: options.controlApiClient ?? (deniedCode ? denyingClient(deniedCode) : client),
fetchFn: options.fetchFn ?? upstream(textDeltas, calls),
lookup,
})
@@ -872,4 +944,112 @@ describe('codex-llm-proxy attempt telemetry', () => {
expect(failureCount(metricsText, code)).toBe(0)
}
)
+
+ it('(k) keeps a silent upstream attempt open with SSE comments and counts them', async () => {
+ const { res, lines } = await run('att-heartbeat', 0, 0, undefined, undefined, {
+ fetchFn: silentThenCompletingUpstream(100),
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ expect(res.status).toBe(200)
+ const firstData = res.text.indexOf('data:')
+ expect(firstData).toBeGreaterThan(0)
+ expect(keepaliveCount(res.text.slice(0, firstData))).toBeGreaterThanOrEqual(2)
+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')
+ expect(res.text).toContain('data: {"type":"done","outcome":"success"}')
+ // Nothing is written after the done frame.
+ expect(res.text.endsWith('data: {"type":"done","outcome":"success"}\n\n')).toBe(true)
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-heartbeat',
+ outcome: 'success',
+ deliveredAs: 'sse_done',
+ heartbeats: keepaliveCount(res.text),
+ })
+ })
+
+ it('(l) sends no keepalive before the redeem succeeds, so a denial keeps its HTTP status', async () => {
+ const denied = slowDenyingClient('no_grant', 60)
+ const { res, lines } = await run('att-heartbeat-denied', 0, 0, undefined, undefined, {
+ controlApiClient: denied.client,
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ // Witness: the redeem ran and outlasted three heartbeat intervals.
+ expect(denied.redeemCalls()).toBe(1)
+ expect(res.status).toBe(403)
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.body).toEqual({ error: 'no_grant' })
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ code: 'no_grant',
+ deliveredAs: 'http_status',
+ httpStatus: 403,
+ heartbeats: 0,
+ })
+ })
+
+ it('(m) delivers an upstream failure after a keepalive as an SSE error frame', async () => {
+ const { res, lines } = await run('att-heartbeat-failed', 0, 0, undefined, undefined, {
+ fetchFn: slowFailingUpstream(60, 500),
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ expect(res.status).toBe(200)
+ expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(
+ true
+ )
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ outcome: 'failed',
+ code: 'provider_unavailable',
+ deliveredAs: 'sse_error',
+ heartbeats: keepaliveCount(res.text),
+ })
+ expect('httpStatus' in lines[0]!).toBe(false)
+ })
+
+ // Node drops a write on a closed response without an error, so a heartbeat
+ // the server never stops is invisible on the wire: it is an interval that
+ // outlives its attempt. The test tracks the timers themselves.
+ it('(n) clears every heartbeat timer once the attempt ends and keeps serving', async () => {
+ const uncaught: unknown[] = []
+ const onUncaught = (err: unknown): void => {
+ uncaught.push(err)
+ }
+ const realSetInterval = globalThis.setInterval
+ const heartbeatTimers: unknown[] = []
+ const setSpy = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+ handler: () => void,
+ ms?: number
+ ) => {
+ const timer = realSetInterval(handler, ms)
+ // 5 ms is the interval this test configures; nothing else uses it.
+ if (ms === 5) heartbeatTimers.push(timer)
+ return timer
+ }) as unknown as typeof setInterval)
+ const clearSpy = vi.spyOn(globalThis, 'clearInterval')
+ process.on('uncaughtException', onUncaught)
+ try {
+ const first = await run('att-heartbeat-end', 0, 0, undefined, undefined, {
+ fetchFn: silentThenCompletingUpstream(30),
+ configOverrides: { heartbeatIntervalMs: 5 },
+ })
+ // Witness: the heartbeat was running while the upstream was silent.
+ expect(keepaliveCount(first.res.text)).toBeGreaterThanOrEqual(1)
+ await new Promise(resolve => setTimeout(resolve, 50))
+ expect(uncaught).toEqual([])
+ const second = await run('att-heartbeat-next', 1, 0, undefined, undefined, {
+ configOverrides: { heartbeatIntervalMs: 5 },
+ })
+ expect(second.res.status).toBe(200)
+ expect(second.res.text).toContain('data: {"type":"done","outcome":"success"}')
+ // Witness: each attempt started its own heartbeat.
+ expect(heartbeatTimers).toHaveLength(2)
+ const cleared = clearSpy.mock.calls.map(call => call[0])
+ for (const timer of heartbeatTimers) expect(cleared).toContain(timer)
+ } finally {
+ process.off('uncaughtException', onUncaught)
+ setSpy.mockRestore()
+ clearSpy.mockRestore()
+ }
+ })
})
diff --git a/codex-llm-proxy/test/sseBackpressure.test.ts b/codex-llm-proxy/test/sseBackpressure.test.ts
index 3cfb1971e..b6c8ef609 100644
--- a/codex-llm-proxy/test/sseBackpressure.test.ts
+++ b/codex-llm-proxy/test/sseBackpressure.test.ts
@@ -93,6 +93,7 @@ function config(): CodexLlmProxyConfig {
maxStreamDurationMs: 60_000,
maxDeadlineMs: 60_000,
upstreamIdleTimeoutMs: 300_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
diff --git a/codex-llm-proxy/test/sseHeartbeat.test.ts b/codex-llm-proxy/test/sseHeartbeat.test.ts
new file mode 100644
index 000000000..25b59764b
--- /dev/null
+++ b/codex-llm-proxy/test/sseHeartbeat.test.ts
@@ -0,0 +1,95 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { EventEmitter } from 'node:events'
+import { startSseHeartbeat } from '../src/sseHeartbeat.js'
+
+class FakeResponse extends EventEmitter {
+ readonly writes: string[] = []
+ writableEnded = false
+ destroyed = false
+ write(chunk: string): boolean {
+ this.writes.push(chunk)
+ return true
+ }
+}
+
+describe('startSseHeartbeat', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('writes one SSE comment per interval, first at +interval, and counts each', () => {
+ const res = new FakeResponse()
+ const beats: number[] = []
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => beats.push(1))
+
+ vi.advanceTimersByTime(999)
+ expect(res.writes).toEqual([])
+ vi.advanceTimersByTime(1)
+ expect(res.writes).toEqual([': keepalive\n\n'])
+ vi.advanceTimersByTime(2_000)
+ expect(res.writes).toHaveLength(3)
+ expect(beats).toHaveLength(3)
+ stop()
+ })
+
+ it('never writes after stop, and never waits on drain', () => {
+ const res = new FakeResponse()
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ // Witness: the heartbeat was running before stop.
+ expect(res.writes).toHaveLength(1)
+ stop()
+ vi.advanceTimersByTime(10_000)
+ expect(res.writes).toHaveLength(1)
+ expect(res.listenerCount('drain')).toBe(0)
+ })
+
+ it.each([
+ [
+ 'the response already ended',
+ (res: FakeResponse): void => {
+ res.writableEnded = true
+ },
+ ],
+ [
+ 'the socket was destroyed',
+ (res: FakeResponse): void => {
+ res.destroyed = true
+ },
+ ],
+ ] as const)('skips the write when %s', (_case, close) => {
+ const res = new FakeResponse()
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ // Witness: the tick runs and writes while the response is open.
+ expect(res.writes).toHaveLength(1)
+ close(res)
+ vi.advanceTimersByTime(3_000)
+ expect(res.writes).toHaveLength(1)
+ stop()
+ })
+
+ it('skips the write once the attempt signal aborted', () => {
+ const res = new FakeResponse()
+ const abort = new AbortController()
+ const stop = startSseHeartbeat(res, abort.signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ expect(res.writes).toHaveLength(1)
+ abort.abort()
+ vi.advanceTimersByTime(3_000)
+ expect(res.writes).toHaveLength(1)
+ stop()
+ })
+
+ it('rejects an interval that is not a positive integer', () => {
+ const res = new FakeResponse()
+ for (const interval of [0, -1, 1.5, Number.NaN]) {
+ expect(() => startSseHeartbeat(res, new AbortController().signal, interval, () => {})).toThrow(
+ /heartbeat interval must be a positive integer/
+ )
+ }
+ })
+})
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index fdb080383..71afe4c83 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -203,8 +203,9 @@ behavior changes:
- It logs one `codex_proxy_attempt_finished` event per completion attempt,
with identifiers and counts only (never the body, ticket, frames, tool
names or arguments): `providerAttemptId`, `hostRef`, `model`,
- `requestHash`, `outcome`, `deliveredAs`, `toolCalls`, `textChunks` and
- `durationMs`. On a stream that reached the upstream's terminal frame,
+ `requestHash`, `outcome`, `deliveredAs`, `toolCalls`, `textChunks`,
+ `heartbeats` and `durationMs`. On a stream that reached the upstream's
+ terminal frame,
`outcome` is `success`, `canceled`, `error` or `unknown`, with
`deliveredAs: 'sse_done'` and `usage` when present. On a thrown failure,
`outcome` is `failed` and the event adds `code`, the transport `reason`,
@@ -212,6 +213,16 @@ behavior changes:
`tool_call_limit_exceeded`) and `deliveredAs`: `http_status` with
`httpStatus` when no SSE byte had been sent, or `sse_error` when the
failure went out as an SSE error frame.
+ - Once the redeem succeeds, the proxy writes a `: keepalive` SSE comment
+ every `CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000) until the
+ response ends. The comments keep the Host's HTTP client, whose headers
+ and body timeouts are 300 s, from cutting an attempt while the upstream
+ is silent (reasoning, or tool calls buffered until the stream completes).
+ SSE readers, including `mcp-host`, ignore comment lines. `heartbeats`
+ counts the comments sent. A keepalive counts as a sent SSE byte, so a
+ failure after the first one is delivered as `sse_error`, not
+ `http_status`. A redeem denial is always `http_status`, because no
+ keepalive is written before the redeem succeeds.
- Do not confuse the two `outcome` fields. The finalize receipt sent to
control-api keeps `success | canceled | error | unknown`. Only the
`codex_proxy_attempt_finished` log line adds `failed`.
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 72d162387..7822e0a83 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -258,6 +258,17 @@ freeze gate checks that every code the proxy constructs is in the fixture's
lower the idle timeout; the transport never raises it above the contract
value. Both cuts are counted in
`grok_proxy_upstream_timeouts_total{kind="idle"|"total"}`.
+- Keepalive: once the redeem succeeds, the proxy writes a `: keepalive` SSE
+ comment every `GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000) until
+ the response ends. The comments keep the Host's HTTP client, whose headers
+ and body timeouts are 300 s, from cutting an attempt while the upstream is
+ silent (reasoning, or tool calls buffered until the stream completes). SSE
+ readers, including `mcp-host`, ignore comment lines. The
+ `grok_proxy_attempt_finished` log line counts them in `heartbeats`. A
+ keepalive counts as a sent SSE byte, so every failure after the first one is
+ delivered as an SSE error frame instead of an HTTP status. A redeem denial
+ always keeps its HTTP status, because no keepalive is written before the
+ redeem succeeds.
- `tool_call_limit_exceeded`: the upstream response carried more than
`maxToolCalls` tool calls. The proxy returns HTTP 422 whose body is the code
alone — `{"error":"tool_call_limit_exceeded"}` — or an SSE error frame when
diff --git a/grok-llm-proxy/src/config.ts b/grok-llm-proxy/src/config.ts
index f1c0253dd..5c309fdd3 100644
--- a/grok-llm-proxy/src/config.ts
+++ b/grok-llm-proxy/src/config.ts
@@ -1,4 +1,5 @@
import { STREAM_LIMITS } from './requestLimits.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
export type GrokLlmProxyConfig = {
runtimePort: number
@@ -9,6 +10,8 @@ export type GrokLlmProxyConfig = {
maxDeadlineMs: number
/** Lowers STREAM_LIMITS.upstreamIdleTimeoutMs; the transport never raises it. */
upstreamIdleTimeoutMs: number
+ /** Interval between SSE keepalive comments once the redeem succeeded. */
+ heartbeatIntervalMs: number
jwtIssuer: string
jwtPublicKey: string
executionEnabled: boolean
@@ -96,6 +99,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCo
env.GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,
STREAM_LIMITS.upstreamIdleTimeoutMs
),
+ heartbeatIntervalMs: requiredPositiveInt(
+ 'GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
+ env.GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
+ DEFAULT_HEARTBEAT_INTERVAL_MS
+ ),
jwtIssuer: env.GROK_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',
jwtPublicKey: requiredPem('GROK_LLM_PROXY_JWT_PUBLIC_KEY', env.GROK_LLM_PROXY_JWT_PUBLIC_KEY),
executionEnabled: env.GROK_LLM_PROXY_EXECUTION_ENABLED === 'true',
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index 43debdcba..fc1878164 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -110,6 +110,12 @@ export type StreamGrokCompletionInput = {
hostRef: string
operation: 'completion_stream'
}) => Promise<RedeemAttemptSuccess>
+ /**
+ * Called once, after the redeem succeeded and its served model and deadline
+ * were accepted, and before the upstream fetch. The server starts its SSE
+ * heartbeat here, so a denied redeem still answers with an HTTP status.
+ */
+ onRedeemed?: () => void
finalize: (input: {
attemptReceipt: string
receipt: {
@@ -166,6 +172,7 @@ export async function streamGrokCompletion(
redeemed.transport.maxStreamDurationMs
)
const idleTimeoutMs = assertBoundedIdleTimeout(input.upstreamIdleTimeoutMs)
+ input.onRedeemed?.()
const accessToken = redeemed.accessToken
let outcome: StreamGrokCompletionResult['outcome'] = 'unknown'
diff --git a/grok-llm-proxy/src/server.ts b/grok-llm-proxy/src/server.ts
index 234f1736a..bccdc8442 100644
--- a/grok-llm-proxy/src/server.ts
+++ b/grok-llm-proxy/src/server.ts
@@ -23,6 +23,7 @@ import {
defaultAddressLookup,
} from './originPolicy.js'
import { RequestLimitError, streamGate } from './requestLimits.js'
+import { startSseHeartbeat } from './sseHeartbeat.js'
const COMPLETION_KEYS = new Set(['executionTicket', 'requestHash', 'request', 'deadlineMs'])
const ADMIN_KEYS = new Set(['accessToken'])
@@ -176,6 +177,10 @@ export function createProxyApps(
const attemptStarted = Date.now()
let toolCalls = 0
let textChunks = 0
+ let heartbeats = 0
+ // Set once the redeem succeeded. Every exit path below calls it before
+ // it ends the response: a tick after `res.end()` would write after end.
+ let stopHeartbeat: (() => void) | undefined
try {
release = await streamGate.acquire(abort.signal)
res.status(200)
@@ -198,6 +203,12 @@ export function createProxyApps(
},
signal: abort.signal,
redeem: input => client.redeem(input),
+ onRedeemed: () => {
+ stopHeartbeat = startSseHeartbeat(res, abort.signal, config.heartbeatIntervalMs, () => {
+ heartbeats += 1
+ })
+ abort.signal.addEventListener('abort', stopHeartbeat, { once: true })
+ },
finalize: input => client.finalize(input),
fetchFn,
lookup,
@@ -207,6 +218,7 @@ export function createProxyApps(
return writeSseChunk(res, `data: ${JSON.stringify(frame)}\n\n`, abort.signal)
},
})
+ stopHeartbeat?.()
res.write(
`data: ${JSON.stringify({ type: 'done', outcome: result.outcome, ...(result.usage ? { usage: result.usage } : {}) })}\n\n`
)
@@ -219,6 +231,7 @@ export function createProxyApps(
deliveredAs: 'sse_done',
toolCalls,
textChunks,
+ heartbeats,
durationMs: Date.now() - attemptStarted,
...(result.usage ? { usage: result.usage } : {}),
}
@@ -226,6 +239,7 @@ export function createProxyApps(
else logger.warn(finished, 'grok attempt finished')
res.end()
} catch (err) {
+ stopHeartbeat?.()
const mapped = mapError(err)
metrics.observeAttempt('error', 'completion_stream')
metrics.observeAttemptFailure(failureLabel(mapped.code))
@@ -246,6 +260,7 @@ export function createProxyApps(
...(deliveredAs === 'http_status' ? { httpStatus: mapped.status } : {}),
toolCalls,
textChunks,
+ heartbeats,
durationMs: Date.now() - attemptStarted,
},
'grok attempt finished'
@@ -261,6 +276,7 @@ export function createProxyApps(
res.setHeader('content-type', 'application/json; charset=utf-8')
res.status(mapped.status).json({ error: mapped.code })
} finally {
+ stopHeartbeat?.()
release?.()
}
})()
diff --git a/grok-llm-proxy/src/sseHeartbeat.ts b/grok-llm-proxy/src/sseHeartbeat.ts
new file mode 100644
index 000000000..06029f4c7
--- /dev/null
+++ b/grok-llm-proxy/src/sseHeartbeat.ts
@@ -0,0 +1,43 @@
+/** The part of a server response the heartbeat touches. */
+export type HeartbeatSink = {
+ write(chunk: string): boolean
+ readonly writableEnded: boolean
+ readonly destroyed: boolean
+}
+
+const KEEPALIVE = ': keepalive\n\n'
+
+/**
+ * Well under the Host's 300 s undici headers/body timeouts, so a silent
+ * upstream never reaches them: queue wait (60 s) + redeem (15 s) + one
+ * interval stays below 300 s.
+ */
+export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
+
+/**
+ * Writes an SSE comment every `intervalMs` so the Host's HTTP client, whose
+ * headers and body timeouts are 300 s, keeps the attempt open while the
+ * upstream is silent (reasoning, or tool calls that the proxy buffers until
+ * the stream completes). SSE readers ignore comment lines.
+ *
+ * The first comment goes out at +intervalMs. A tick never waits on `drain`
+ * and never adds a listener: a comment is 13 bytes, and a slow client is
+ * already bounded by the frame writer's backpressure. The caller must call
+ * the returned stop function before ending the response.
+ */
+export function startSseHeartbeat(
+ res: HeartbeatSink,
+ signal: AbortSignal,
+ intervalMs: number,
+ onBeat: () => void
+): () => void {
+ if (!Number.isInteger(intervalMs) || intervalMs <= 0) {
+ throw new Error('heartbeat interval must be a positive integer')
+ }
+ const timer = setInterval(() => {
+ if (res.writableEnded || res.destroyed || signal.aborted) return
+ res.write(KEEPALIVE)
+ onBeat()
+ }, intervalMs)
+ return () => clearInterval(timer)
+}
diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.ts
index 60fbcc063..a682b1d0b 100644
--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts
+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts
@@ -803,6 +803,68 @@ describe('streamGrokCompletion', () => {
).rejects.toMatchObject({ code: 'origin_denied' })
})
+ it('calls onRedeemed once, after a matching redeem and before the upstream fetch', async () => {
+ const ticket = {
+ jti: 'jti-redeemed',
+ hostRef: 'research-host',
+ model: REQUEST.model,
+ requestHash: REQUEST_HASH,
+ providerAttemptId: 'att-redeemed',
+ }
+ const finalize = vi.fn(async () => ({
+ providerAttemptId: 'att-redeemed',
+ outcome: 'success' as const,
+ duplicate: false,
+ }))
+ const order: string[] = []
+ await streamGrokCompletion({
+ executionTicket: 'ticket-redeemed',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket,
+ redeem: async () => {
+ order.push('redeem')
+ return redeemSuccess()
+ },
+ onRedeemed: () => order.push('onRedeemed'),
+ finalize,
+ fetchFn: vi.fn(async () => {
+ order.push('fetch')
+ return sseResponse(['data: {"type":"response.completed","response":{"usage":{}}}\n\n'])
+ }),
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ })
+ expect(order).toEqual(['redeem', 'onRedeemed', 'fetch'])
+
+ // A denied redeem and a served-model mismatch never start the heartbeat.
+ const denied = vi.fn(async (): Promise<RedeemAttemptSuccess> => {
+ throw new Error('no_grant')
+ })
+ const mismatched = vi.fn(async () =>
+ redeemSuccess({ transport: { ...redeemSuccess().transport, servedModel: 'other' } })
+ )
+ const onRedeemed = vi.fn()
+ for (const redeem of [denied, mismatched]) {
+ await expect(
+ streamGrokCompletion({
+ executionTicket: 'ticket-redeemed',
+ requestHash: REQUEST_HASH,
+ request: REQUEST,
+ ticket,
+ redeem,
+ onRedeemed,
+ finalize,
+ fetchFn: vi.fn(),
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ })
+ ).rejects.toThrow()
+ }
+ // Witness: both redeems ran, so the path that could call onRedeemed was entered.
+ expect(denied).toHaveBeenCalledTimes(1)
+ expect(mismatched).toHaveBeenCalledTimes(1)
+ expect(onRedeemed).not.toHaveBeenCalled()
+ })
+
it('follows one frozen same-origin redirect then streams', async () => {
const fetchFn = vi.fn(async () => {
if (fetchFn.mock.calls.length === 1) {
diff --git a/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts b/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
index 10be35337..8ea677f32 100644
--- a/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
+++ b/grok-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
@@ -79,6 +79,7 @@ function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfig
maxStreamDurationMs: 30_000,
maxDeadlineMs: 30_000,
upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index a0dc2b079..2271f44c6 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -35,6 +35,7 @@ function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfig
maxStreamDurationMs: 300_000,
maxDeadlineMs: 300_000,
upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
@@ -320,6 +321,23 @@ describe('grok-llm-proxy security surface', () => {
}).upstreamIdleTimeoutMs
).toBe(45_000)
})
+
+ it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
+ const env = {
+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ }
+ expect(loadConfig(env).heartbeatIntervalMs).toBe(15_000)
+ expect(
+ loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '5000' }).heartbeatIntervalMs
+ ).toBe(5_000)
+ for (const raw of ['0', '-1']) {
+ expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
+ /GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be a finite integer greater than zero/
+ )
+ }
+ })
})
describe('grok-llm-proxy execution kill switch', () => {
@@ -695,6 +713,59 @@ describe('grok-llm-proxy attempt telemetry', () => {
return line ? Number(line.split(' ').pop()) : 0
}
+ function keepaliveCount(text: string): number {
+ return text.split(': keepalive\n\n').length - 1
+ }
+
+ // Stays silent for `silentMs` after the fetch resolves, then streams one text
+ // delta and completes, like a model that reasons before its first token.
+ function silentThenCompletingUpstream(silentMs: number): typeof fetch {
+ return (async () =>
+ new Response(
+ new ReadableStream<Uint8Array>({
+ async start(controller) {
+ await new Promise(resolve => setTimeout(resolve, silentMs))
+ controller.enqueue(
+ new TextEncoder().encode(
+ 'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
+ 'data: {"type":"response.completed"}\n\n'
+ )
+ )
+ controller.close()
+ },
+ }),
+ { status: 200, headers: { 'content-type': 'text/event-stream' } }
+ )) as typeof fetch
+ }
+
+ // Answers `status` with no SSE body after `delayMs`.
+ function slowFailingUpstream(delayMs: number, status: number): typeof fetch {
+ return (async () => {
+ await new Promise(resolve => setTimeout(resolve, delayMs))
+ return new Response('upstream failed', { status })
+ }) as typeof fetch
+ }
+
+ // Denies the redeem after `delayMs`, which is longer than the heartbeat
+ // interval the caller configures.
+ function slowDenyingClient(code: string, delayMs: number): {
+ client: ControlApiClient
+ redeemCalls: () => number
+ } {
+ let calls = 0
+ const client = {
+ async redeem() {
+ calls += 1
+ await new Promise(resolve => setTimeout(resolve, delayMs))
+ throw new ControlApiClientError(code, 'control API request denied')
+ },
+ async finalize() {
+ throw new Error('finalize must not run after a denied redeem')
+ },
+ } as unknown as ControlApiClient
+ return { client, redeemCalls: () => calls }
+ }
+
async function run(options: {
providerAttemptId: string
textDeltas?: number
@@ -704,12 +775,15 @@ describe('grok-llm-proxy attempt telemetry', () => {
fetchFn?: typeof fetch
maxStreamDurationMs?: number
configOverrides?: Partial<GrokLlmProxyConfig>
+ controlApiClient?: ControlApiClient
}) {
const info = vi.spyOn(logger, 'info')
const warn = vi.spyOn(logger, 'warn')
const { client, receipts } = grantingClient(options.maxStreamDurationMs)
const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {
- controlApiClient: options.deniedCode ? denyingClient(options.deniedCode) : client,
+ controlApiClient:
+ options.controlApiClient ??
+ (options.deniedCode ? denyingClient(options.deniedCode) : client),
fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),
lookup,
})
@@ -1014,4 +1088,117 @@ describe('grok-llm-proxy attempt telemetry', () => {
expect(failureCount(metricsText, code)).toBe(0)
}
)
+
+ it('(hb1) keeps a silent upstream attempt open with SSE comments and counts them', async () => {
+ const { res, lines } = await run({
+ providerAttemptId: 'att-heartbeat',
+ fetchFn: silentThenCompletingUpstream(100),
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ expect(res.status).toBe(200)
+ const firstData = res.text.indexOf('data:')
+ expect(firstData).toBeGreaterThan(0)
+ expect(keepaliveCount(res.text.slice(0, firstData))).toBeGreaterThanOrEqual(2)
+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')
+ // Nothing is written after the done frame.
+ expect(res.text.endsWith('data: {"type":"done","outcome":"success"}\n\n')).toBe(true)
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-heartbeat',
+ outcome: 'success',
+ deliveredAs: 'sse_done',
+ heartbeats: keepaliveCount(res.text),
+ })
+ })
+
+ it('(hb2) sends no keepalive before the redeem succeeds, so a denial keeps its HTTP status', async () => {
+ const denied = slowDenyingClient('no_grant', 60)
+ const { res, lines } = await run({
+ providerAttemptId: 'att-heartbeat-denied',
+ controlApiClient: denied.client,
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ // Witness: the redeem ran and outlasted three heartbeat intervals.
+ expect(denied.redeemCalls()).toBe(1)
+ expect(res.status).toBe(403)
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.body).toEqual({ error: 'no_grant' })
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ code: 'no_grant',
+ deliveredAs: 'http_status',
+ httpStatus: 403,
+ heartbeats: 0,
+ })
+ })
+
+ it('(hb3) delivers an upstream failure after a keepalive as an SSE error frame', async () => {
+ const { res, lines } = await run({
+ providerAttemptId: 'att-heartbeat-failed',
+ fetchFn: slowFailingUpstream(60, 500),
+ configOverrides: { heartbeatIntervalMs: 20 },
+ })
+ expect(res.status).toBe(200)
+ expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
+ expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(
+ true
+ )
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ outcome: 'failed',
+ code: 'provider_unavailable',
+ deliveredAs: 'sse_error',
+ heartbeats: keepaliveCount(res.text),
+ })
+ expect('httpStatus' in lines[0]!).toBe(false)
+ })
+
+ // Node drops a write on a closed response without an error, so a heartbeat
+ // the server never stops is invisible on the wire: it is an interval that
+ // outlives its attempt. The test tracks the timers themselves.
+ it('(hb4) clears every heartbeat timer once the attempt ends and keeps serving', async () => {
+ const uncaught: unknown[] = []
+ const onUncaught = (err: unknown): void => {
+ uncaught.push(err)
+ }
+ const realSetInterval = globalThis.setInterval
+ const heartbeatTimers: unknown[] = []
+ const setSpy = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
+ handler: () => void,
+ ms?: number
+ ) => {
+ const timer = realSetInterval(handler, ms)
+ // 5 ms is the interval this test configures; nothing else uses it.
+ if (ms === 5) heartbeatTimers.push(timer)
+ return timer
+ }) as unknown as typeof setInterval)
+ const clearSpy = vi.spyOn(globalThis, 'clearInterval')
+ process.on('uncaughtException', onUncaught)
+ try {
+ const first = await run({
+ providerAttemptId: 'att-heartbeat-end',
+ fetchFn: silentThenCompletingUpstream(30),
+ configOverrides: { heartbeatIntervalMs: 5 },
+ })
+ // Witness: the heartbeat was running while the upstream was silent.
+ expect(keepaliveCount(first.res.text)).toBeGreaterThanOrEqual(1)
+ await new Promise(resolve => setTimeout(resolve, 50))
+ expect(uncaught).toEqual([])
+ const second = await run({
+ providerAttemptId: 'att-heartbeat-next',
+ textDeltas: 1,
+ configOverrides: { heartbeatIntervalMs: 5 },
+ })
+ expect(second.res.status).toBe(200)
+ expect(second.res.text).toContain('data: {"type":"done","outcome":"success"}')
+ // Witness: each attempt started its own heartbeat.
+ expect(heartbeatTimers).toHaveLength(2)
+ const cleared = clearSpy.mock.calls.map(call => call[0])
+ for (const timer of heartbeatTimers) expect(cleared).toContain(timer)
+ } finally {
+ process.off('uncaughtException', onUncaught)
+ setSpy.mockRestore()
+ clearSpy.mockRestore()
+ }
+ })
})
diff --git a/grok-llm-proxy/test/sseBackpressure.test.ts b/grok-llm-proxy/test/sseBackpressure.test.ts
index 52771b2c8..656c33e85 100644
--- a/grok-llm-proxy/test/sseBackpressure.test.ts
+++ b/grok-llm-proxy/test/sseBackpressure.test.ts
@@ -93,6 +93,7 @@ function config(): GrokLlmProxyConfig {
maxStreamDurationMs: 60_000,
maxDeadlineMs: 60_000,
upstreamIdleTimeoutMs: 600_000,
+ heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
diff --git a/grok-llm-proxy/test/sseHeartbeat.test.ts b/grok-llm-proxy/test/sseHeartbeat.test.ts
new file mode 100644
index 000000000..25b59764b
--- /dev/null
+++ b/grok-llm-proxy/test/sseHeartbeat.test.ts
@@ -0,0 +1,95 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { EventEmitter } from 'node:events'
+import { startSseHeartbeat } from '../src/sseHeartbeat.js'
+
+class FakeResponse extends EventEmitter {
+ readonly writes: string[] = []
+ writableEnded = false
+ destroyed = false
+ write(chunk: string): boolean {
+ this.writes.push(chunk)
+ return true
+ }
+}
+
+describe('startSseHeartbeat', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('writes one SSE comment per interval, first at +interval, and counts each', () => {
+ const res = new FakeResponse()
+ const beats: number[] = []
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => beats.push(1))
+
+ vi.advanceTimersByTime(999)
+ expect(res.writes).toEqual([])
+ vi.advanceTimersByTime(1)
+ expect(res.writes).toEqual([': keepalive\n\n'])
+ vi.advanceTimersByTime(2_000)
+ expect(res.writes).toHaveLength(3)
+ expect(beats).toHaveLength(3)
+ stop()
+ })
+
+ it('never writes after stop, and never waits on drain', () => {
+ const res = new FakeResponse()
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ // Witness: the heartbeat was running before stop.
+ expect(res.writes).toHaveLength(1)
+ stop()
+ vi.advanceTimersByTime(10_000)
+ expect(res.writes).toHaveLength(1)
+ expect(res.listenerCount('drain')).toBe(0)
+ })
+
+ it.each([
+ [
+ 'the response already ended',
+ (res: FakeResponse): void => {
+ res.writableEnded = true
+ },
+ ],
+ [
+ 'the socket was destroyed',
+ (res: FakeResponse): void => {
+ res.destroyed = true
+ },
+ ],
+ ] as const)('skips the write when %s', (_case, close) => {
+ const res = new FakeResponse()
+ const stop = startSseHeartbeat(res, new AbortController().signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ // Witness: the tick runs and writes while the response is open.
+ expect(res.writes).toHaveLength(1)
+ close(res)
+ vi.advanceTimersByTime(3_000)
+ expect(res.writes).toHaveLength(1)
+ stop()
+ })
+
+ it('skips the write once the attempt signal aborted', () => {
+ const res = new FakeResponse()
+ const abort = new AbortController()
+ const stop = startSseHeartbeat(res, abort.signal, 1_000, () => {})
+ vi.advanceTimersByTime(1_000)
+ expect(res.writes).toHaveLength(1)
+ abort.abort()
+ vi.advanceTimersByTime(3_000)
+ expect(res.writes).toHaveLength(1)
+ stop()
+ })
+
+ it('rejects an interval that is not a positive integer', () => {
+ const res = new FakeResponse()
+ for (const interval of [0, -1, 1.5, Number.NaN]) {
+ expect(() => startSseHeartbeat(res, new AbortController().signal, interval, () => {})).toThrow(
+ /heartbeat interval must be a positive integer/
+ )
+ }
+ })
+})
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index cd947ff9e..31e677e92 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 3dc925d7b..826596dc8 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.368",
+ "version": "0.1.369",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
index 8552ec53f..51e96ed03 100644
--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
@@ -166,6 +166,41 @@ describe('CodexLlmProxyClient', () => {
expect(classifyFailoverClass(idle.code, idle.retryable)).toBe('provider_unavailable')
})
+ // The proxy writes `: keepalive` SSE comments while the upstream is silent,
+ // before and between data frames, and a comment can straddle two reads.
+ it('ignores proxy keepalive comments around and between data frames', async () => {
+ const encoder = new TextEncoder()
+ const chunks = [
+ ': keepalive\n\n: keep',
+ 'alive\n\n',
+ 'data: {"type":"text","text":"hel"}\n\n: keepalive\n\n',
+ 'data: {"type":"text","text":"lo"}\n\n',
+ ': keepalive\n\ndata: {"type":"done","outcome":"success"}\n\n',
+ ]
+ const fetchFn = vi.fn().mockResolvedValue({
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const chunk of chunks) controller.enqueue(encoder.encode(chunk))
+ controller.close()
+ },
+ }),
+ })
+ const client = new CodexLlmProxyClient({
+ runtimeUrl: resolveCodexProxyRuntimeUrl('http://codex-llm-proxy:8080'),
+ readPlatformJwt: () => 'platform-jwt',
+ fetchFn: fetchFn as unknown as typeof fetch,
+ })
+ const result = await client.stream({
+ executionTicket: 'ticket-123456',
+ requestHash: 'a'.repeat(64),
+ request: { model: 'gpt-5.3-codex' },
+ })
+ expect(result.text).toBe('hello')
+ expect(result.toolCalls).toEqual([])
+ expect(result.outcome).toBe('success')
+ })
+
it('refuses a runtime URL that is not absolute', () => {
expect(
() =>
diff --git a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
index 739357322..ade34df88 100644
--- a/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/grokLlmProxyClient.test.ts
@@ -67,6 +67,32 @@ describe('GrokLlmProxyClient', () => {
expect(result.usage).toEqual({ inputTokens: 3, outputTokens: 5 })
})
+ // The proxy writes `: keepalive` SSE comments while the upstream is silent,
+ // before and between data frames, and a comment can straddle two reads.
+ it('ignores proxy keepalive comments around and between data frames', async () => {
+ const encoder = new TextEncoder()
+ const chunks = [
+ ': keepalive\n\n: keep',
+ 'alive\n\n',
+ 'data: {"type":"text","text":"hel"}\n\n: keepalive\n\n',
+ 'data: {"type":"text","text":"lo"}\n\n',
+ ': keepalive\n\ndata: {"type":"done","outcome":"success"}\n\n',
+ ]
+ const fetchFn = vi.fn().mockResolvedValue({
+ ok: true,
+ body: new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (const chunk of chunks) controller.enqueue(encoder.encode(chunk))
+ controller.close()
+ },
+ }),
+ })
+ const result = await client(fetchFn).stream(STREAM_INPUT)
+ expect(result.text).toBe('hello')
+ expect(result.toolCalls).toEqual([])
+ expect(result.outcome).toBe('success')
+ })
+
it('refuses a runtime URL that is not absolute', () => {
expect(
() =>