Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(llm-proxy): address review R1 on stream limits

Alfredo Lopez03ccd87d98f3
diff --git a/codex-llm-proxy/src/config.ts b/codex-llm-proxy/src/config.ts
index ccf498677..03097dbf6 100644
--- a/codex-llm-proxy/src/config.ts
+++ b/codex-llm-proxy/src/config.ts
@@ -1,6 +1,6 @@
import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import { STREAM_LIMITS } from './requestLimits.js'
-import { DEFAULT_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
export type CodexLlmProxyConfig = {
runtimePort: number
@@ -33,6 +33,11 @@ function requiredPositiveInt(name: string, raw: string | undefined, fallback?: n
return value
}
+function atMost(name: string, value: number, max: number): number {
+ if (value > max) throw new Error(`${name} must be at most ${max}`)
+ return value
+}
+
function requiredPem(name: string, raw: string | undefined): string {
const value = raw?.trim() ?? ''
if (!value.includes('BEGIN') || !value.includes('KEY')) {
@@ -111,10 +116,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyC
env.CODEX_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,
STREAM_LIMITS.upstreamIdleTimeoutMs
),
- heartbeatIntervalMs: requiredPositiveInt(
+ // Refused rather than lowered: the operator's value is either applied or
+ // the proxy does not start.
+ heartbeatIntervalMs: atMost(
'CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
- env.CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
- DEFAULT_HEARTBEAT_INTERVAL_MS
+ requiredPositiveInt(
+ 'CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
+ env.CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
+ DEFAULT_HEARTBEAT_INTERVAL_MS
+ ),
+ MAX_HEARTBEAT_INTERVAL_MS
),
jwtIssuer: env.CODEX_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',
jwtPublicKey: requiredPem('CODEX_LLM_PROXY_JWT_PUBLIC_KEY', env.CODEX_LLM_PROXY_JWT_PUBLIC_KEY),
diff --git a/codex-llm-proxy/src/metrics.ts b/codex-llm-proxy/src/metrics.ts
index a92fab9c5..2e79ee5ff 100644
--- a/codex-llm-proxy/src/metrics.ts
+++ b/codex-llm-proxy/src/metrics.ts
@@ -10,7 +10,8 @@ export function createProxyMetrics(register: Registry) {
const streamSeconds = new Histogram({
name: 'codex_llm_proxy_stream_duration_seconds',
help: 'Codex proxy stream duration',
- buckets: [0.1, 0.5, 1, 2, 5, 15, 30, 60, 120, 300],
+ // Up to the 1800 s stream cap, so no in-range stream lands only in +Inf.
+ buckets: [0.1, 0.5, 1, 2, 5, 15, 30, 60, 120, 300, 600, 1200, 1800],
registers: [register],
})
const attemptFailures = new Counter({
diff --git a/codex-llm-proxy/src/requestLimits.ts b/codex-llm-proxy/src/requestLimits.ts
index bb88e8dee..237b8738d 100644
--- a/codex-llm-proxy/src/requestLimits.ts
+++ b/codex-llm-proxy/src/requestLimits.ts
@@ -34,6 +34,11 @@ export class RequestLimitError extends Error {
}
export function assertBoundedDeadline(deadlineMs: number | undefined, maxDeadlineMs: number): number {
+ // Without a valid maximum, Math.min below would return NaN for a valid
+ // request deadline instead of refusing it.
+ if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
+ throw new RequestLimitError('deadline is invalid')
+ }
const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
throw new RequestLimitError('deadline is invalid')
diff --git a/codex-llm-proxy/src/sseHeartbeat.ts b/codex-llm-proxy/src/sseHeartbeat.ts
index 06029f4c7..52b315b7c 100644
--- a/codex-llm-proxy/src/sseHeartbeat.ts
+++ b/codex-llm-proxy/src/sseHeartbeat.ts
@@ -14,6 +14,14 @@ const KEEPALIVE = ': keepalive\n\n'
*/
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
+/**
+ * Largest interval config accepts. Two queue waits (visual gate, then stream
+ * gate: 120 s), the 15 s redeem and one interval must still end before the
+ * Host's 300 s headers timeout, and one interval must stay below its body
+ * timeout.
+ */
+export const MAX_HEARTBEAT_INTERVAL_MS = 60_000
+
/**
* Writes an SSE comment every `intervalMs` so the Host's HTTP client, whose
* headers and body timeouts are 300 s, keeps the attempt open while the
diff --git a/codex-llm-proxy/test/metrics.test.ts b/codex-llm-proxy/test/metrics.test.ts
new file mode 100644
index 000000000..86bfec243
--- /dev/null
+++ b/codex-llm-proxy/test/metrics.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest'
+import { Registry } from 'prom-client'
+import { createProxyMetrics } from '../src/metrics.js'
+import { STREAM_LIMITS } from '../src/requestLimits.js'
+
+type Bucket = { le: number; count: number }
+
+async function streamDurationBuckets(register: Registry): Promise<Bucket[]> {
+ const metric = (await register.getMetricsAsJSON()).find(
+ entry => entry.name === 'codex_llm_proxy_stream_duration_seconds'
+ )
+ expect(metric, 'stream duration histogram must be registered').toBeDefined()
+ return (metric!.values as Array<{ metricName?: string; labels: { le?: number | string }; value: number }>)
+ .filter(value => value.metricName === 'codex_llm_proxy_stream_duration_seconds_bucket')
+ // prom-client labels the overflow bucket '+Inf', which Number() reads as NaN.
+ .map(value => ({
+ le: value.labels.le === '+Inf' ? Infinity : Number(value.labels.le),
+ count: value.value,
+ }))
+}
+
+describe('stream duration histogram', () => {
+ it('resolves every duration up to the stream cap into a finite bucket', async () => {
+ const register = new Registry()
+ const metrics = createProxyMetrics(register)
+ // A stream between the old 300 s cap and the current one.
+ metrics.observeStream(1_500_000)
+ const buckets = await streamDurationBuckets(register)
+ const finite = buckets.filter(bucket => Number.isFinite(bucket.le))
+ // Liveness witness: the observation reached the histogram.
+ expect(buckets.find(bucket => bucket.le === Infinity)?.count).toBe(1)
+ expect(Math.max(...finite.map(bucket => bucket.le))).toBeGreaterThanOrEqual(
+ STREAM_LIMITS.maxStreamDurationMs / 1000
+ )
+ expect(buckets.find(bucket => bucket.le === 1200)?.count).toBe(0)
+ expect(buckets.find(bucket => bucket.le === 1800)?.count).toBe(1)
+ })
+})
diff --git a/codex-llm-proxy/test/requestLimits.test.ts b/codex-llm-proxy/test/requestLimits.test.ts
index c7e9fe13b..b06dc6f84 100644
--- a/codex-llm-proxy/test/requestLimits.test.ts
+++ b/codex-llm-proxy/test/requestLimits.test.ts
@@ -1,12 +1,13 @@
import { describe, expect, it } from 'vitest'
import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
import {
+ assertBoundedDeadline,
RequestLimitError,
STREAM_LIMITS,
StreamGate,
VISUAL_STREAM_LIMITS,
} from '../src/requestLimits.js'
-import { DEFAULT_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
// undici's default headersTimeout and bodyTimeout in the Node 24.16.0 image the
// Host runs; the Host sets neither for its proxy calls.
@@ -20,15 +21,35 @@ describe('stream timing invariant', () => {
// so both must end, and one heartbeat interval pass, inside the timeout.
// A visual request can wait twice: on visualStreamGate before the body is
// parsed, then on streamGate when the parsed body no longer needs the
- // visual slot. Both gates share the maxQueueWaitMs default.
+ // visual slot. Both gates share the maxQueueWaitMs default. The interval
+ // is configurable, so the largest one config accepts is the one pinned.
+ expect(MAX_HEARTBEAT_INTERVAL_MS).toBe(60_000)
+ expect(DEFAULT_HEARTBEAT_INTERVAL_MS).toBeLessThanOrEqual(MAX_HEARTBEAT_INTERVAL_MS)
expect(
2 * STREAM_LIMITS.maxQueueWaitMs +
CONTROL_API_REQUEST_TIMEOUT_MS +
- DEFAULT_HEARTBEAT_INTERVAL_MS
+ MAX_HEARTBEAT_INTERVAL_MS
).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)
})
})
+describe('assertBoundedDeadline', () => {
+ it('refuses a missing or invalid proxy maximum even when the request carries a deadline', () => {
+ // Liveness witness: the same request deadline passes with a valid maximum.
+ expect(assertBoundedDeadline(1_000, 1_800_000)).toBe(1_000)
+ for (const maxDeadlineMs of [undefined, Number.NaN, 0, -1, 1.5]) {
+ let caught: unknown
+ try {
+ assertBoundedDeadline(1_000, maxDeadlineMs as unknown as number)
+ } catch (err) {
+ caught = err
+ }
+ expect(caught, `maxDeadlineMs=${String(maxDeadlineMs)}`).toBeInstanceOf(RequestLimitError)
+ expect((caught as Error).message).toBe('deadline is invalid')
+ }
+ })
+})
+
describe('StreamGate', () => {
it('admits a queued waiter once a running stream releases', async () => {
const gate = new StreamGate(1, 1)
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index d392f17cc..4c3d2005b 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -438,6 +438,24 @@ describe('codex-llm-proxy security surface', () => {
}
})
+ it('refuses at startup a heartbeat interval the Host HTTP client would time out on', () => {
+ const env = {
+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ CODEX_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ CODEX_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ }
+ // The bound itself is accepted, so the refusal below is the bound and not
+ // a parse failure.
+ expect(
+ loadConfig({ ...env, CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '60000' }).heartbeatIntervalMs
+ ).toBe(60_000)
+ for (const raw of ['60001', '600000']) {
+ expect(() => loadConfig({ ...env, CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
+ /CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be at most 60000/
+ )
+ }
+ })
+
it('refuses a visual envelope budget below the shared contract', () => {
const required = {
CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
diff --git a/codex-llm-proxy/test/sseHeartbeat.test.ts b/codex-llm-proxy/test/sseHeartbeat.test.ts
index 25b59764b..7f139de11 100644
--- a/codex-llm-proxy/test/sseHeartbeat.test.ts
+++ b/codex-llm-proxy/test/sseHeartbeat.test.ts
@@ -1,6 +1,30 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { EventEmitter } from 'node:events'
-import { startSseHeartbeat } from '../src/sseHeartbeat.js'
+import { readFileSync } from 'node:fs'
+import {
+ DEFAULT_HEARTBEAT_INTERVAL_MS,
+ MAX_HEARTBEAT_INTERVAL_MS,
+ startSseHeartbeat,
+} from '../src/sseHeartbeat.js'
+
+const architectureDoc = new URL(
+ '../../docs/architecture/codex-subscription-transport-contract.md',
+ import.meta.url
+)
+
+describe('heartbeat interval documentation', () => {
+ it('states the default and the maximum interval config accepts', () => {
+ const doc = readFileSync(architectureDoc, 'utf8')
+ const stated = /`CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS`\s+\(default (\d+), at most (\d+)\)/.exec(
+ doc
+ )
+ expect(stated, 'the doc must state the heartbeat default and maximum').not.toBeNull()
+ expect({ default: Number(stated![1]), max: Number(stated![2]) }).toEqual({
+ default: DEFAULT_HEARTBEAT_INTERVAL_MS,
+ max: MAX_HEARTBEAT_INTERVAL_MS,
+ })
+ })
+})
class FakeResponse extends EventEmitter {
readonly writes: string[] = []
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 0056e37ad..211f524b5 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.603",
+ "version": "0.1.604",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.603",
+ "version": "0.1.604",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 4bcfb5ef1..64d00e152 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.603",
+ "version": "0.1.604",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/services.llmProviderAttemptEnvelope.test.ts b/control-api/test/services.llmProviderAttemptEnvelope.test.ts
index d0a883882..c6f287fb7 100644
--- a/control-api/test/services.llmProviderAttemptEnvelope.test.ts
+++ b/control-api/test/services.llmProviderAttemptEnvelope.test.ts
@@ -18,25 +18,20 @@ import { CODEX_EXECUTION_TICKET_TTL_SECONDS } from '../src/services/llmProviderA
// cap. The budget reservation and the in-flight usage grace must both outlast
// that, or spend arrives after the reservation freed the budget headroom.
describe('provider attempt lifetime envelope', () => {
- it('derives each attempt lifetime from the ticket TTL and the redeem stream cap', () => {
- expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(
- CODEX_EXECUTION_TICKET_TTL_SECONDS * 1000 + CODEX_MAX_STREAM_DURATION_MS
- )
- expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(
- GROK_EXECUTION_TICKET_TTL_SECONDS * 1000 + GROK_MAX_STREAM_DURATION_MS
- )
+ it('sizes each attempt lifetime as the ticket TTL plus the redeem stream cap', () => {
+ // Concrete inputs, read from the modules the ticket issuer and the redeem
+ // use, so the lifetime below is checked against values, not the formula.
+ expect(CODEX_EXECUTION_TICKET_TTL_SECONDS).toBe(60)
+ expect(GROK_EXECUTION_TICKET_TTL_SECONDS).toBe(60)
+ expect(CODEX_MAX_STREAM_DURATION_MS).toBe(1_800_000)
+ expect(GROK_MAX_STREAM_DURATION_MS).toBe(1_800_000)
expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)
expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)
})
it('keeps each attempt reservation alive for the attempt plus the rollup margin', () => {
- const margin = config.budgetReservationTtlSeconds
- expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(
- Math.ceil(CODEX_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin
- )
- expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(
- Math.ceil(GROK_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin
- )
+ // The task-level reservation TTL is the rollup margin added to the lifetime.
+ expect(config.budgetReservationTtlSeconds).toBe(300)
expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
})
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index c8f9b792e..d2c829060 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -328,8 +328,9 @@ behavior changes:
`httpStatus` when no SSE byte had been sent, or `sse_error` when the
failure went out as an SSE error frame.
- Once the redeem succeeds, the proxy writes a `: keepalive` SSE comment
- every `CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000) until the
- response ends. The comments keep the Host's HTTP client, whose headers
+ every `CODEX_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000, at most 60000) until the
+ response ends. A larger value stops the proxy at startup instead of
+ being lowered. The comments keep the Host's HTTP client, whose headers
and body timeouts are 300 s, from cutting an attempt while the upstream
is silent (reasoning, or tool calls buffered until the stream completes).
SSE readers, including `mcp-host`, ignore comment lines. `heartbeats`
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index 30d8d34bd..e01f67ecb 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -127,6 +127,25 @@ message, not the conversation, and the 1:4 spread between the two numbers is a
design choice rather than an arithmetic requirement: a turn of N calls adds
N+1 messages, so a full 256-call turn occupies 257 of the 1024 message slots.
+| Limit | Value |
+| --------------------- | ------- |
+| maxRequestBodyBytes | 1048576 |
+| maxMessages | 1024 |
+| maxToolCalls | 256 |
+| maxOutputTokens | 16384 |
+| maxStreamDurationMs | 1800000 |
+| maxDeadlineMs | 1800000 |
+| maxConcurrentStreams | 8 |
+| maxQueuedRequests | 16 |
+| maxQueueWaitMs | 60000 |
+| upstreamIdleTimeoutMs | 600000 |
+| maxRetriesPerAttempt | 1 |
+
+The limit values live only in the table above, which
+`grok-llm-proxy/test/contractFreeze.test.ts` checks against the fixture; the
+same suite pins the fixture to the contract `LIMITS` and the proxy
+`STREAM_LIMITS`.
+
All three enforcement points read this module — the control-api authorizer,
`grok-llm-proxy` and the Host — so a deployment that mixes versions rejects
requests that fall between the old and the new bounds. Which code the caller
@@ -168,12 +187,11 @@ Proxy robustness (both proxies):
checks the abort signal before redeeming a ticket. It also rejects an
invalid or out-of-bounds deadline before the redeem, so the single-use
ticket is not consumed.
-- A stream-gate waiter still queued after `maxQueueWaitMs` (60 s) is rejected
+- A stream-gate waiter still queued after `maxQueueWaitMs` is rejected
with `provider_unavailable` (reason `stream queue wait exceeded`). Queue
wait, the 15 s control-api redeem timeout and the first keepalive together
stay below the Host HTTP client's 300 s header timeout.
-- A single attempt streams for at most `maxStreamDurationMs` (1 800 000 ms):
- the minimum of the proxy configuration, `STREAM_LIMITS`, the contract
+- A single attempt streams for at most `maxStreamDurationMs`: the minimum of the proxy configuration, `STREAM_LIMITS`, the contract
`maxDeadlineMs` and the value control-api returns on redeem.
- The proxy fails at startup when `GROK_LLM_PROXY_CONTROL_API_URL` or
`GROK_LLM_PROXY_CONTROL_API_TOKEN` is empty.
@@ -260,7 +278,7 @@ freeze gate checks that every code the proxy constructs is in the fixture's
It is not retryable and not failover-eligible: another attempt would spend
the same budget on the same turn.
- Idle timeout: when the upstream sends no byte for `upstreamIdleTimeoutMs`
- (600000, the value read from the Grok Build client source), the proxy cancels the upstream
+ (the value read from the Grok Build client source), the proxy cancels the upstream
body and fails the attempt with `provider_unavailable` (HTTP 503, reason
`upstream stream idle timeout`). That code stays retryable and
failover-eligible, because a silent upstream is an outage of that provider,
@@ -269,8 +287,9 @@ freeze gate checks that every code the proxy constructs is in the fixture's
value. Both cuts are counted in
`grok_proxy_upstream_timeouts_total{kind="idle"|"total"}`.
- Keepalive: once the redeem succeeds, the proxy writes a `: keepalive` SSE
- comment every `GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000) until
- the response ends. The comments keep the Host's HTTP client, whose headers
+ comment every `GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS` (default 15000, at most 60000) until
+ the response ends. A larger value stops the proxy at startup instead of
+ being lowered. The comments keep the Host's HTTP client, whose headers
and body timeouts are 300 s, from cutting an attempt while the upstream is
silent (reasoning, or tool calls buffered until the stream completes). SSE
readers, including `mcp-host`, ignore comment lines. The
diff --git a/grok-llm-proxy/src/config.ts b/grok-llm-proxy/src/config.ts
index 333874e58..ed72472d1 100644
--- a/grok-llm-proxy/src/config.ts
+++ b/grok-llm-proxy/src/config.ts
@@ -1,5 +1,5 @@
import { STREAM_LIMITS } from './requestLimits.js'
-import { DEFAULT_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from './sseHeartbeat.js'
export type GrokLlmProxyConfig = {
runtimePort: number
@@ -31,6 +31,11 @@ function requiredPositiveInt(name: string, raw: string | undefined, fallback?: n
return value
}
+function atMost(name: string, value: number, max: number): number {
+ if (value > max) throw new Error(`${name} must be at most ${max}`)
+ return value
+}
+
function requiredPem(name: string, raw: string | undefined): string {
const value = raw?.trim() ?? ''
if (!value.includes('BEGIN') || !value.includes('KEY')) {
@@ -99,10 +104,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): GrokLlmProxyCo
env.GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS,
STREAM_LIMITS.upstreamIdleTimeoutMs
),
- heartbeatIntervalMs: requiredPositiveInt(
+ // Refused rather than lowered: the operator's value is either applied or
+ // the proxy does not start.
+ heartbeatIntervalMs: atMost(
'GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
- env.GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
- DEFAULT_HEARTBEAT_INTERVAL_MS
+ requiredPositiveInt(
+ 'GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS',
+ env.GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS,
+ DEFAULT_HEARTBEAT_INTERVAL_MS
+ ),
+ MAX_HEARTBEAT_INTERVAL_MS
),
jwtIssuer: env.GROK_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',
jwtPublicKey: requiredPem('GROK_LLM_PROXY_JWT_PUBLIC_KEY', env.GROK_LLM_PROXY_JWT_PUBLIC_KEY),
diff --git a/grok-llm-proxy/src/metrics.ts b/grok-llm-proxy/src/metrics.ts
index 50590cf30..955a0718d 100644
--- a/grok-llm-proxy/src/metrics.ts
+++ b/grok-llm-proxy/src/metrics.ts
@@ -10,7 +10,8 @@ export function createProxyMetrics(register: Registry) {
const streamSeconds = new Histogram({
name: 'grok_llm_proxy_stream_duration_seconds',
help: 'Grok proxy stream duration',
- buckets: [0.1, 0.5, 1, 2, 5, 15, 30, 60, 120, 300],
+ // Up to the 1800 s stream cap, so no in-range stream lands only in +Inf.
+ buckets: [0.1, 0.5, 1, 2, 5, 15, 30, 60, 120, 300, 600, 1200, 1800],
registers: [register],
})
const attemptFailures = new Counter({
diff --git a/grok-llm-proxy/src/requestLimits.ts b/grok-llm-proxy/src/requestLimits.ts
index 5b2f70a28..c6c9d304c 100644
--- a/grok-llm-proxy/src/requestLimits.ts
+++ b/grok-llm-proxy/src/requestLimits.ts
@@ -25,6 +25,11 @@ export function assertBoundedDeadline(
deadlineMs: number | undefined,
maxDeadlineMs: number
): number {
+ // Without a valid maximum, Math.min below would return NaN for a valid
+ // request deadline instead of refusing it.
+ if (!Number.isInteger(maxDeadlineMs) || maxDeadlineMs <= 0) {
+ throw new RequestLimitError('deadline is invalid')
+ }
const requested = deadlineMs ?? Math.min(maxDeadlineMs, STREAM_LIMITS.maxStreamDurationMs)
if (!Number.isFinite(requested) || !Number.isInteger(requested) || requested <= 0) {
throw new RequestLimitError('deadline is invalid')
diff --git a/grok-llm-proxy/src/sseHeartbeat.ts b/grok-llm-proxy/src/sseHeartbeat.ts
index 06029f4c7..1ea9ba011 100644
--- a/grok-llm-proxy/src/sseHeartbeat.ts
+++ b/grok-llm-proxy/src/sseHeartbeat.ts
@@ -14,6 +14,13 @@ const KEEPALIVE = ': keepalive\n\n'
*/
export const DEFAULT_HEARTBEAT_INTERVAL_MS = 15_000
+/**
+ * Largest interval config accepts. The queue wait (60 s), the 15 s redeem and
+ * one interval must still end before the Host's 300 s headers timeout, and
+ * one interval must stay below its body timeout.
+ */
+export const MAX_HEARTBEAT_INTERVAL_MS = 60_000
+
/**
* Writes an SSE comment every `intervalMs` so the Host's HTTP client, whose
* headers and body timeouts are 300 s, keeps the attempt open while the
diff --git a/grok-llm-proxy/test/contractFreeze.test.ts b/grok-llm-proxy/test/contractFreeze.test.ts
index 9cd4d2825..caeeae590 100644
--- a/grok-llm-proxy/test/contractFreeze.test.ts
+++ b/grok-llm-proxy/test/contractFreeze.test.ts
@@ -24,6 +24,10 @@ const fixturePath = join(
here,
'../../tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json'
)
+const architectureDocPath = join(
+ here,
+ '../../docs/architecture/grok-subscription-transport-contract.md'
+)
// The limits the frozen fixture publishes, exactly.
const FIXTURE_LIMIT_KEYS = [
@@ -72,6 +76,30 @@ function readFixture(): Fixture {
return JSON.parse(readFileSync(fixturePath, 'utf8')) as Fixture
}
+/**
+ * Rows of the architecture doc's `| Limit | Value |` table, in document order.
+ * The table ends at the first line that is not a table row.
+ */
+function parseLimitsTable(doc: string): Array<[string, string]> {
+ const lines = doc.split('\n')
+ const header = lines.findIndex(line => /^\|\s*Limit\s*\|\s*Value\s*\|$/.test(line))
+ expect(
+ header,
+ 'architecture doc must contain a "| Limit | Value |" table'
+ ).toBeGreaterThanOrEqual(0)
+ expect(lines[header + 1], 'limits table header must be followed by a separator row').toMatch(
+ /^\|\s*-+\s*\|\s*-+\s*\|$/
+ )
+ const rows: Array<[string, string]> = []
+ for (const line of lines.slice(header + 2)) {
+ if (!line.startsWith('|')) break
+ const cells = /^\|\s*([A-Za-z0-9]+)\s*\|\s*(\S+)\s*\|$/.exec(line)
+ expect(cells, `malformed limits table row: ${line}`).not.toBeNull()
+ rows.push([cells![1]!, cells![2]!])
+ }
+ return rows
+}
+
const srcDir = join(here, '../src')
// Every construction of a transport error in src, with its first code argument.
@@ -151,6 +179,24 @@ describe('grok-subscription contract freeze', () => {
expect(limits.maxRetriesPerAttempt).toBe(1)
})
+ it('pins the architecture doc limits table to the limits the fixture publishes', () => {
+ const { limits } = readFixture()
+ // A row that drifts from the fixture misdescribes what the runtime
+ // enforces. The table is parsed as a whole, so a duplicated, extra or
+ // missing row also fails.
+ const docLimits = parseLimitsTable(readFileSync(architectureDocPath, 'utf8'))
+ expect(
+ docLimits.map(([key]) => key).sort(),
+ 'architecture doc limits table must list each fixture limit exactly once'
+ ).toEqual([...FIXTURE_LIMIT_KEYS].sort())
+ for (const [key, value] of docLimits) {
+ expect(
+ { [key]: value },
+ `architecture doc limits table must list ${key} = ${String(limits[key])}`
+ ).toEqual({ [key]: String(limits[key]) })
+ }
+ })
+
it('pins the contract LIMITS to the limits the fixture publishes', () => {
const { limits } = readFixture()
const runtimeLimits: Record<string, number> = { ...LIMITS }
diff --git a/grok-llm-proxy/test/metrics.test.ts b/grok-llm-proxy/test/metrics.test.ts
new file mode 100644
index 000000000..b595b4fa6
--- /dev/null
+++ b/grok-llm-proxy/test/metrics.test.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest'
+import { Registry } from 'prom-client'
+import { createProxyMetrics } from '../src/metrics.js'
+import { STREAM_LIMITS } from '../src/requestLimits.js'
+
+type Bucket = { le: number; count: number }
+
+async function streamDurationBuckets(register: Registry): Promise<Bucket[]> {
+ const metric = (await register.getMetricsAsJSON()).find(
+ entry => entry.name === 'grok_llm_proxy_stream_duration_seconds'
+ )
+ expect(metric, 'stream duration histogram must be registered').toBeDefined()
+ return (metric!.values as Array<{ metricName?: string; labels: { le?: number | string }; value: number }>)
+ .filter(value => value.metricName === 'grok_llm_proxy_stream_duration_seconds_bucket')
+ // prom-client labels the overflow bucket '+Inf', which Number() reads as NaN.
+ .map(value => ({
+ le: value.labels.le === '+Inf' ? Infinity : Number(value.labels.le),
+ count: value.value,
+ }))
+}
+
+describe('stream duration histogram', () => {
+ it('resolves every duration up to the stream cap into a finite bucket', async () => {
+ const register = new Registry()
+ const metrics = createProxyMetrics(register)
+ // A stream between the old 300 s cap and the current one.
+ metrics.observeStream(1_500_000)
+ const buckets = await streamDurationBuckets(register)
+ const finite = buckets.filter(bucket => Number.isFinite(bucket.le))
+ // Liveness witness: the observation reached the histogram.
+ expect(buckets.find(bucket => bucket.le === Infinity)?.count).toBe(1)
+ expect(Math.max(...finite.map(bucket => bucket.le))).toBeGreaterThanOrEqual(
+ STREAM_LIMITS.maxStreamDurationMs / 1000
+ )
+ expect(buckets.find(bucket => bucket.le === 1200)?.count).toBe(0)
+ expect(buckets.find(bucket => bucket.le === 1800)?.count).toBe(1)
+ })
+})
diff --git a/grok-llm-proxy/test/requestLimits.test.ts b/grok-llm-proxy/test/requestLimits.test.ts
index deb988815..dcee1d5f6 100644
--- a/grok-llm-proxy/test/requestLimits.test.ts
+++ b/grok-llm-proxy/test/requestLimits.test.ts
@@ -1,7 +1,12 @@
import { describe, expect, it } from 'vitest'
import { CONTROL_API_REQUEST_TIMEOUT_MS } from '../src/controlApiClient.js'
-import { RequestLimitError, STREAM_LIMITS, StreamGate } from '../src/requestLimits.js'
-import { DEFAULT_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
+import {
+ assertBoundedDeadline,
+ RequestLimitError,
+ STREAM_LIMITS,
+ StreamGate,
+} from '../src/requestLimits.js'
+import { DEFAULT_HEARTBEAT_INTERVAL_MS, MAX_HEARTBEAT_INTERVAL_MS } from '../src/sseHeartbeat.js'
// undici's default headersTimeout and bodyTimeout in the Node 24.16.0 image the
// Host runs; the Host sets neither for its proxy calls.
@@ -13,12 +18,33 @@ describe('stream timing invariant', () => {
expect(STREAM_LIMITS.maxQueueWaitMs).toBe(60_000)
// Nothing is written while a request waits for a slot or for the redeem,
// so both must end, and one heartbeat interval pass, inside the timeout.
+ // The interval is configurable, so the largest one config accepts is the
+ // one pinned.
+ expect(MAX_HEARTBEAT_INTERVAL_MS).toBe(60_000)
+ expect(DEFAULT_HEARTBEAT_INTERVAL_MS).toBeLessThanOrEqual(MAX_HEARTBEAT_INTERVAL_MS)
expect(
- STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + DEFAULT_HEARTBEAT_INTERVAL_MS
+ STREAM_LIMITS.maxQueueWaitMs + CONTROL_API_REQUEST_TIMEOUT_MS + MAX_HEARTBEAT_INTERVAL_MS
).toBeLessThan(HOST_UNDICI_TIMEOUT_MS)
})
})
+describe('assertBoundedDeadline', () => {
+ it('refuses a missing or invalid proxy maximum even when the request carries a deadline', () => {
+ // Liveness witness: the same request deadline passes with a valid maximum.
+ expect(assertBoundedDeadline(1_000, 1_800_000)).toBe(1_000)
+ for (const maxDeadlineMs of [undefined, Number.NaN, 0, -1, 1.5]) {
+ let caught: unknown
+ try {
+ assertBoundedDeadline(1_000, maxDeadlineMs as unknown as number)
+ } catch (err) {
+ caught = err
+ }
+ expect(caught, `maxDeadlineMs=${String(maxDeadlineMs)}`).toBeInstanceOf(RequestLimitError)
+ expect((caught as Error).message).toBe('deadline is invalid')
+ }
+ })
+})
+
describe('StreamGate', () => {
it('admits a queued waiter once a running stream releases', async () => {
const gate = new StreamGate(1, 1)
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 9d305c47b..46a2a36d3 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -349,6 +349,24 @@ describe('grok-llm-proxy security surface', () => {
)
}
})
+
+ it('refuses at startup a heartbeat interval the Host HTTP client would time out on', () => {
+ const env = {
+ GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
+ GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
+ }
+ // The bound itself is accepted, so the refusal below is the bound and not
+ // a parse failure.
+ expect(
+ loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '60000' }).heartbeatIntervalMs
+ ).toBe(60_000)
+ for (const raw of ['60001', '600000']) {
+ expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
+ /GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be at most 60000/
+ )
+ }
+ })
})
describe('grok-llm-proxy execution kill switch', () => {
diff --git a/grok-llm-proxy/test/sseHeartbeat.test.ts b/grok-llm-proxy/test/sseHeartbeat.test.ts
index 25b59764b..68504c46d 100644
--- a/grok-llm-proxy/test/sseHeartbeat.test.ts
+++ b/grok-llm-proxy/test/sseHeartbeat.test.ts
@@ -1,6 +1,30 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { EventEmitter } from 'node:events'
-import { startSseHeartbeat } from '../src/sseHeartbeat.js'
+import { readFileSync } from 'node:fs'
+import {
+ DEFAULT_HEARTBEAT_INTERVAL_MS,
+ MAX_HEARTBEAT_INTERVAL_MS,
+ startSseHeartbeat,
+} from '../src/sseHeartbeat.js'
+
+const architectureDoc = new URL(
+ '../../docs/architecture/grok-subscription-transport-contract.md',
+ import.meta.url
+)
+
+describe('heartbeat interval documentation', () => {
+ it('states the default and the maximum interval config accepts', () => {
+ const doc = readFileSync(architectureDoc, 'utf8')
+ const stated = /`GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS`\s+\(default (\d+), at most (\d+)\)/.exec(
+ doc
+ )
+ expect(stated, 'the doc must state the heartbeat default and maximum').not.toBeNull()
+ expect({ default: Number(stated![1]), max: Number(stated![2]) }).toEqual({
+ default: DEFAULT_HEARTBEAT_INTERVAL_MS,
+ max: MAX_HEARTBEAT_INTERVAL_MS,
+ })
+ })
+})
class FakeResponse extends EventEmitter {
readonly writes: string[] = []