Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import { generateKeyPairSync } from 'node:crypto'
import { createServer, request as httpRequest } from 'node:http'
import type { AddressInfo } from 'node:net'
import request from 'supertest'
import {
ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,
LIMITS,
hashGrokCompletionRequest,
parseGrokCompletionRequest,
parseGrokCompletionRequestV1,
} from '@clerum/grok-provider-attempt-contract'
import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'
import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'
import { type GrokLlmProxyConfig, loadConfig } from '../src/config.js'
import {
ControlApiClient,
ControlApiClientError,
type FinalizeAttemptSuccess,
type RedeemAttemptSuccess,
} from '../src/controlApiClient.js'
import { GrokTransportError, MAX_TOOL_CALL_ARGUMENT_BYTES } from '../src/grokTransport.js'
import { REDACT_PATHS, logger } from '../src/logger.js'
import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
import {
BodyBudget,
DEFAULT_MAX_BODY_BYTES,
ENVELOPE_ALLOWANCE_BYTES,
RequestLimitError,
STREAM_LIMITS,
VISUAL_STREAM_LIMITS,
streamGate,
visualStreamGate,
} from '../src/requestLimits.js'
import { createProxyApps } from '../src/server.js'
const { privateKey, publicKey } = generateKeyPairSync('rsa', {
modulusLength: 2048,
publicKeyEncoding: { type: 'spki', format: 'pem' },
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
})
function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfig {
return {
runtimePort: 8080,
adminPort: 8081,
probePort: 9090,
maxBodyBytes: 1024,
maxVisualBodyBytes: LIMITS.maxVisualRequestBodyBytes,
maxStreamDurationMs: 1_800_000,
maxDeadlineMs: 1_800_000,
upstreamIdleTimeoutMs: 600_000,
heartbeatIntervalMs: 15_000,
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
controlApiBaseUrl: '',
controlApiServiceName: 'grok-llm-proxy',
controlApiServiceToken: '',
...overrides,
}
}
function sign(payload: Record<string, unknown>, audience: string): string {
return jwt.sign(payload, privateKey, {
algorithm: 'RS256',
issuer: 'control-api',
audience,
expiresIn: 60,
})
}
function platformToken(overrides: Record<string, unknown> = {}): string {
return sign(
{
sub: 'default/research-host',
hostRefs: ['research-host'],
workflowControlScopes: ['llm:grok:execute'],
scope: 'workflow:approval:request',
...overrides,
},
'workflow-approvals'
)
}
function ticket(): string {
return sign(
{
jti: '11111111-1111-4111-8111-111111111111',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash: 'a'.repeat(64),
providerAttemptId: 'att-1',
},
'grok-llm-proxy'
)
}
function adminPermit(): string {
return sign(
{ sub: 'admin-1', typ: 'grok-admin-permit', operation: 'catalog_list' },
'grok-llm-proxy-admin'
)
}
describe('grok-llm-proxy security surface', () => {
it('exposes only the frozen runtime, admin, and probe routes', async () => {
const { runtimeApp, adminApp, probeApp } = createProxyApps(config())
expect((await request(probeApp).get('/healthz')).status).toBe(200)
expect((await request(probeApp).get('/readyz')).status).toBe(200)
const metrics = await request(probeApp).get('/metrics')
expect(metrics.status).toBe(200)
expect(metrics.text).not.toMatch(/account|refresh|accessToken/i)
expect((await request(runtimeApp).get('/internal/runtime/v1/grok/completions')).status).toBe(
404
)
expect((await request(adminApp).get('/internal/admin/v1/grok/models')).status).toBe(404)
})
it('denies an admin permit on the runtime listener and a runtime ticket on admin', async () => {
const { runtimeApp, adminApp } = createProxyApps(config())
const runtime = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${adminPermit()}`)
.send({ executionTicket: ticket(), requestHash: 'a'.repeat(64), request: {} })
expect(runtime.status).toBe(403)
expect(runtime.body.error).toBe('insufficient_scope')
const admin = await request(adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${ticket()}`)
.send({})
expect(admin.status).toBe(403)
expect(admin.body.error).toBe('insufficient_scope')
})
it('rejects unknown fields, invalid deadlines, and incorrect content types', async () => {
const { runtimeApp } = createProxyApps(config())
const unknown = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
extra: true,
})
expect(unknown.body.error).toBe('unknown_field')
const deadline = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
deadlineMs: 1_800_001,
})
expect(deadline.body.error).toBe('invalid_request')
const ctype = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.set('Content-Type', 'text/plain')
.send('not-json')
expect(ctype.status).toBe(415)
})
it('rejects an oversized body before invoking the completion path', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: 32 }))
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: { pad: 'x'.repeat(200) },
})
expect(res.status).toBe(413)
})
it('(g1-2) rate limits the completion endpoint with a JSON rate_limited body', async () => {
const { runtimeApp } = createProxyApps(config())
const completion = () => request(runtimeApp).post('/internal/runtime/v1/grok/completions')
// Within the window every request reaches the platform JWT gate: the witness
// that the 61st rejection comes from the limiter and not from that gate.
for (let i = 0; i < 60; i += 1) {
const accepted = await completion().send({})
expect(accepted.status).toBe(401)
}
const limited = await completion().send({})
expect(limited.status).toBe(429)
// G1-2 (#720): the Host reads the JSON error code; a text body would fall
// back to provider_unavailable ("Model Overloaded").
expect(limited.headers['content-type']).toMatch(/^application\/json/)
expect(limited.body).toEqual({ error: 'rate_limited' })
// The library's own headers stay: Retry-After and the draft-7 pair.
expect(limited.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)
expect(limited.headers['ratelimit-policy']).toBe('60;w=60')
})
it('reserves the visual transport budget for an authenticated ~30 MiB V2 body', async () => {
const { runtimeApp, adminApp } = createProxyApps(
config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES })
)
// Deliberately invalid ticket: this test checks parser admission and the
// unchanged ticket gate, without redeeming or contacting any model. 30 MiB
// is past the Codex 24 MiB visual ceiling and inside the Grok 35 MiB one.
const payload = {
executionTicket: 'invalid-ticket',
requestHash: 'a'.repeat(64),
request: {
schemaVersion: 'grok-completion-request.v2',
messages: [
{
role: 'user',
contentParts: [{ type: 'image', data: 'A'.repeat(30 * 1024 * 1024) }],
},
],
},
}
expect(Buffer.byteLength(JSON.stringify(payload))).toBeGreaterThan(24 * 1024 * 1024)
const acquire = vi.spyOn(visualStreamGate, 'acquire')
const admitted = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send(payload)
expect(admitted.status).toBe(403)
expect(admitted.body.error).toBe('ticket_invalid')
// Witness: the body went through the visual gate, not the ordinary parser.
expect(acquire).toHaveBeenCalledTimes(1)
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
// R9-M-B: without a platform JWT the body is never read, so the caller
// gets 401 instead of a parser's 413. The 401 is written before the upload
// ends; a keep-alive client has the unread rest discarded.
const anonymous = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Connection', 'keep-alive')
.send(payload)
expect(anonymous.status).toBe(401)
const noScope = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Connection', 'keep-alive')
.set('Authorization', `Bearer ${platformToken({ workflowControlScopes: [] })}`)
.send(payload)
expect(noScope.status).toBe(401)
const v1 = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
...payload,
request: { ...payload.request, schemaVersion: 'grok-completion-request.v1' },
})
expect(v1.status).toBe(413)
expect(v1.body.error).toBe('payload_too_large')
const admin = await request(adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${adminPermit()}`)
.send(payload)
expect(admin.status).toBe(413)
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
acquire.mockRestore()
}, 30_000)
// r10 Y1: the ticket is a signed JWT of a few hundred bytes. The non-image
// ceiling alone lets an ~8 MiB ticket reach jwt.verify, so the schema bounds
// it at the envelope allowance.
it('T-Y1-grok bounds the execution ticket at the envelope allowance before verifying it', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
const send = (executionTicket: string, body: Record<string, unknown> = {}) =>
request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({ executionTicket, requestHash: 'a'.repeat(64), request: body })
// Witness: a ticket exactly at the bound reaches the verifier.
const atBound = await send('x'.repeat(ENVELOPE_ALLOWANCE_BYTES))
expect(atBound.status).toBe(403)
expect(atBound.body.error).toBe('ticket_invalid')
const overBound = await send('x'.repeat(ENVELOPE_ALLOWANCE_BYTES + 1))
expect(overBound.status).toBe(400)
expect(overBound.body.error).toBe('invalid_request')
// A V2 body pushed past the ordinary cap by its ticket and one small image
// takes the visual gate, is refused at the schema, and frees its slot.
const acquire = vi.spyOn(visualStreamGate, 'acquire')
const visual = await send('x'.repeat(8_000_000), {
schemaVersion: 'grok-completion-request.v2',
messages: [
{ role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },
],
})
expect(visual.status).toBe(400)
expect(visual.body.error).toBe('invalid_request')
expect(acquire).toHaveBeenCalledTimes(1)
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
acquire.mockRestore()
}, 30_000)
it('refuses a declared length past the visual ceiling with 413 before reading it', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
const pad = 'A'.repeat(LIMITS.maxVisualRequestBodyBytes)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.set('Connection', 'keep-alive')
.send({
executionTicket: 'invalid-ticket',
requestHash: 'a'.repeat(64),
request: { schemaVersion: 'grok-completion-request.v2', pad },
})
expect(res.status).toBe(413)
expect(res.body.error).toBe('payload_too_large')
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
}, 30_000)
it('answers a transport payload_too_large with HTTP 413 before any SSE byte', async () => {
const raw = {
schemaVersion: 'grok-completion-request.v2',
requestId: 'req-payload-too-large',
idempotencyKey: 'idem-payload-too-large',
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'hello' }],
}
const parsed = parseGrokCompletionRequest(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
const executionTicket = sign(
{
jti: '12121212-1212-4121-8121-121212121212',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId: 'att-payload-too-large',
},
'grok-llm-proxy'
)
let streamCalls = 0
const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {
streamCompletion: async () => {
streamCalls += 1
throw new GrokTransportError(
'payload_too_large',
'request exceeds maxVisualRequestBodyBytes'
)
},
})
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({ executionTicket, requestHash, request: raw })
// Witness: the request reached the transport.
expect(streamCalls).toBe(1)
expect(res.status).toBe(413)
expect(res.body).toEqual({ error: 'payload_too_large' })
const metricsText = (await request(probeApp).get('/metrics')).text
expect(metricsText).toMatch(
/^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m
)
})
it('does not let a V2 declaration raise the non-image budget to 35 MiB', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
// One byte of text past the non-image ceiling plus its envelope allowance,
// far under the 35 MiB visual ceiling the V2 declaration opens.
const pad = 'x'.repeat(LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES + 1)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
executionTicket: 'invalid-ticket',
requestHash: 'a'.repeat(64),
request: { schemaVersion: 'grok-completion-request.v2', pad },
})
expect(res.status).toBe(413)
expect(res.body.error).toBe('payload_too_large')
// Witness: the same body with an image-sized pad inside contentParts
// instead of text passes the size checks and stops at the ticket gate.
const imageSized = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({
executionTicket: 'invalid-ticket',
requestHash: 'a'.repeat(64),
request: {
schemaVersion: 'grok-completion-request.v2',
messages: [{ role: 'user', contentParts: [{ type: 'image', data: pad }] }],
},
})
expect(imageSized.status).toBe(403)
expect(imageSized.body.error).toBe('ticket_invalid')
}, 30_000)
// Review R3-L3: V2 text repeated in content and in its text parts counts
// once on the non-image budget, as the same text does in V1.
it('R3-L3 counts V2 text repeated in content and its text parts once', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
const completion = (content: string, textPart: string) => ({
executionTicket: 'invalid-ticket',
requestHash: 'a'.repeat(64),
request: {
schemaVersion: 'grok-completion-request.v2',
messages: [
{
role: 'user',
content,
contentParts: [
{ type: 'text', text: textPart },
{ type: 'image', mimeType: 'image/png', data: 'iVBORw0KGgo=' },
],
},
],
},
})
const post = (content: string, textPart: string) =>
request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send(completion(content, textPart))
// The share the proxy measures: no ticket, no image data, and the text
// part blanked because it repeats content.
const { executionTicket: _ticket, ...empty } = completion('', '')
empty.request.messages[0].contentParts[1] = { type: 'image', mimeType: 'image/png', data: '' }
const budget = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
const exact = budget - Buffer.byteLength(JSON.stringify(empty), 'utf8')
// A 4.5 MiB prompt beside an image passes the size checks and stops at the
// ticket gate.
const text = 'x'.repeat(4.5 * 1024 * 1024)
const repeated = await post(text, text)
expect(repeated.status).toBe(403)
expect(repeated.body).toEqual({ error: 'ticket_invalid' })
// Exact boundary: the largest text reaches the ticket gate, one more byte
// is refused 413.
const atLimit = await post('x'.repeat(exact), 'x'.repeat(exact))
expect(atLimit.status).toBe(403)
expect(atLimit.body).toEqual({ error: 'ticket_invalid' })
const overLimit = await post('x'.repeat(exact + 1), 'x'.repeat(exact + 1))
expect(overLimit.status).toBe(413)
expect(overLimit.body).toEqual({ error: 'payload_too_large' })
// Parts that do not repeat content keep both copies and are refused.
const mismatched = await post(`${text.slice(1)}y`, text)
expect(mismatched.status).toBe(413)
expect(mismatched.body).toEqual({ error: 'payload_too_large' })
}, 60_000)
it('rejects a platform JWT whose hostRefs do not bind the ticket hostRef', async () => {
const { runtimeApp } = createProxyApps(config())
const foreign = sign(
{
sub: 'default/other-host',
hostRefs: ['other-host'],
workflowControlScopes: ['llm:grok:execute'],
scope: 'workflow:approval:request',
},
'workflow-approvals'
)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${foreign}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
})
expect(res.status).toBe(403)
expect(res.body.error).toBe('host_binding_mismatch')
})
it('rejects a refresh JWT that otherwise carries llm:grok:execute', async () => {
const { runtimeApp } = createProxyApps(config())
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken({ scope: 'workflow:approval:refresh' })}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
})
expect(res.status).toBe(401)
expect(res.body.error).toBe('Unauthorized')
})
it('rejects a platform JWT that omits the access-token scope', async () => {
const { runtimeApp } = createProxyApps(config())
const missingScope = sign(
{
sub: 'default/research-host',
hostRefs: ['research-host'],
workflowControlScopes: ['llm:grok:execute'],
},
'workflow-approvals'
)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${missingScope}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
})
expect(res.status).toBe(401)
expect(res.body.error).toBe('Unauthorized')
})
it('rejects a platform JWT whose hostRefs is a wildcard', async () => {
const { runtimeApp } = createProxyApps(config())
const wildcard = sign(
{
sub: 'default/research-host',
hostRefs: ['*'],
workflowControlScopes: ['llm:grok:execute'],
scope: 'workflow:approval:request',
},
'workflow-approvals'
)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${wildcard}`)
.send({
executionTicket: ticket(),
requestHash: 'a'.repeat(64),
request: {},
})
expect(res.status).toBe(401)
})
it('rejects execution tickets and admin permits without a numeric exp', () => {
const cfg = config()
const ticketNoExp = jwt.sign(
{
jti: '11111111-1111-4111-8111-111111111111',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash: 'a'.repeat(64),
providerAttemptId: 'att-1',
},
privateKey,
{ algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
)
const permitNoExp = jwt.sign(
{ sub: 'admin-1', typ: 'grok-admin-permit', operation: 'catalog_list' },
privateKey,
{ algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy-admin' }
)
expect(verifyExecutionTicket(ticketNoExp, cfg)).toBeNull()
expect(verifyAdminPermit(permitNoExp, cfg)).toBeNull()
})
// R17-2: body admission can hold a body for up to maxQueueWaitMs before the
// ticket is read, the same span as the ticket TTL, so an honest request can
// reach this gate with an expired ticket. That answer must be the retryable
// ticket_expired; every other ticket failure stays ticket_invalid.
it('T-R17-2-grok answers ticket_expired for an authentic expired ticket and ticket_invalid otherwise', async () => {
const { runtimeApp } = createProxyApps(config({ executionEnabled: false }))
const { privateKey: foreignKey } = generateKeyPairSync('rsa', {
modulusLength: 2048,
publicKeyEncoding: { type: 'spki', format: 'pem' },
privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
})
const expiredClaims = {
jti: '11111111-1111-4111-8111-111111111111',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash: 'a'.repeat(64),
providerAttemptId: 'att-1',
exp: Math.floor(Date.now() / 1000) - 5,
}
const expiredTicket = (
claims: Record<string, unknown>,
key: string = privateKey,
audience = 'grok-llm-proxy'
) => jwt.sign(claims, key, { algorithm: 'RS256', issuer: 'control-api', audience })
const send = (executionTicket: string) =>
request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({ executionTicket, requestHash: 'a'.repeat(64), request: {} })
const expired = await send(expiredTicket(expiredClaims))
expect(expired.status).toBe(403)
expect(expired.body).toEqual({ error: 'ticket_expired' })
// jsonwebtoken checks exp before audience and claims, so each of these is
// expired too; none is an authentic Grok ticket.
for (const forged of [
expiredTicket(expiredClaims, foreignKey),
expiredTicket(expiredClaims, privateKey, 'codex-llm-proxy'),
expiredTicket({ ...expiredClaims, typ: 'codex-execution-ticket' }),
]) {
const refused = await send(forged)
expect(refused.status).toBe(403)
expect(refused.body).toEqual({ error: 'ticket_invalid' })
}
// Witness: a live ticket passes the ticket gate and stops at the next one.
const live = await send(ticket())
expect(live.status).toBe(404)
expect(live.body).toEqual({ error: 'disabled' })
})
it('rejects an admin permit whose operation does not match the route', async () => {
const { adminApp } = createProxyApps(config())
const wrong = sign(
{ sub: 'admin-1', typ: 'grok-admin-permit', operation: 'connection_test' },
'grok-llm-proxy-admin'
)
const res = await request(adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${wrong}`)
.send({ accessToken: 'tok' })
expect(res.status).toBe(401)
})
it('rejects missing, zero, and unbounded config', () => {
expect(() => loadConfig({ GROK_LLM_PROXY_RUNTIME_PORT: '0' })).toThrow(/greater than zero/)
expect(() =>
loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_MAX_BODY_BYTES: '0',
})
).toThrow(/greater than zero/)
expect(() => loadConfig({})).toThrow(/PEM-encoded public key/)
expect(() =>
loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_MAX_STREAM_DURATION_MS: String(Number.MAX_SAFE_INTEGER),
})
).toThrow(/bounded positive integer/)
expect(() =>
loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '0',
})
).toThrow(/GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS must be a finite integer greater than zero/)
})
it('defaults the upstream idle timeout to the published STREAM_LIMITS value', () => {
const loaded = loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
})
expect(loaded.upstreamIdleTimeoutMs).toBe(600_000)
expect(
loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '45000',
}).upstreamIdleTimeoutMs
).toBe(45_000)
})
it('defaults the total stream cap and the deadline ceiling to 30 min', () => {
const loaded = loadConfig({
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
})
expect(loaded.maxStreamDurationMs).toBe(1_800_000)
expect(loaded.maxDeadlineMs).toBe(1_800_000)
})
it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
const env = {
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
}
expect(loadConfig(env).heartbeatIntervalMs).toBe(15_000)
expect(
loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '5000' }).heartbeatIntervalMs
).toBe(5_000)
for (const raw of ['0', '-1']) {
expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
/GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be a finite integer greater than zero/
)
}
})
it('refuses at startup a heartbeat interval the Host HTTP client would time out on', () => {
const env = {
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
}
// The bound itself is accepted, so the refusal below is the bound and not
// a parse failure.
expect(
loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '60000' }).heartbeatIntervalMs
).toBe(60_000)
for (const raw of ['60001', '600000']) {
expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
/GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be at most 60000/
)
}
})
})
describe('grok-llm-proxy execution kill switch', () => {
const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
function validCompletionBody(): Record<string, unknown> {
const raw = {
schemaVersion: 'grok-completion-request.v1',
requestId: 'req-kill-switch',
idempotencyKey: 'idem-kill-switch',
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'hello' }],
}
const parsed = parseGrokCompletionRequestV1(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
const executionTicket = sign(
{
jti: '33333333-3333-4333-8333-333333333333',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId: 'att-kill-switch',
},
'grok-llm-proxy'
)
return { executionTicket, requestHash, request: raw }
}
function recordingClient(): { client: ControlApiClient; redeems: unknown[] } {
const redeems: unknown[] = []
const client = {
async redeem(input: unknown) {
redeems.push(input)
throw new ControlApiClientError('no_grant', 'no grant in kill-switch test')
},
async finalize() {
throw new Error('finalize must not run in kill-switch test')
},
} as unknown as ControlApiClient
return { client, redeems }
}
it('returns 404 disabled for a fully valid runtime JWT + ticket when execution is disabled', async () => {
const body = validCompletionBody()
// Liveness witness: the identical request reaches redeem when enabled.
const enabled = recordingClient()
const live = createProxyApps(config({ executionEnabled: true, maxBodyBytes: 65_536 }), {
controlApiClient: enabled.client,
lookup,
})
const reached = await request(live.runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send(body)
expect(reached.status).toBe(403)
expect(JSON.parse(reached.text)).toEqual({ error: 'no_grant' })
expect(enabled.redeems).toHaveLength(1)
const disabled = recordingClient()
const off = createProxyApps(config({ executionEnabled: false, maxBodyBytes: 65_536 }), {
controlApiClient: disabled.client,
lookup,
})
const res = await request(off.runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send(body)
expect(res.status).toBe(404)
expect(res.body).toEqual({ error: 'disabled' })
expect(disabled.redeems).toHaveLength(0)
})
it('returns 404 disabled for a valid admin permit when execution is disabled', async () => {
const upstream = () =>
new Response(JSON.stringify({ data: [{ id: 'gpt-5.1' }] }), {
status: 200,
headers: { 'content-type': 'application/json' },
})
// Liveness witness: the identical admin call reaches the catalog upstream when enabled.
let enabledCalls = 0
const live = createProxyApps(config({ executionEnabled: true, maxBodyBytes: 65_536 }), {
fetchFn: (async () => {
enabledCalls += 1
return upstream()
}) as typeof fetch,
lookup,
})
const reached = await request(live.adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${adminPermit()}`)
.send({ accessToken: 'tok' })
expect(reached.status).toBe(200)
expect(reached.body.outcome).toBe('ready')
expect(enabledCalls).toBe(1)
let disabledCalls = 0
const off = createProxyApps(config({ executionEnabled: false, maxBodyBytes: 65_536 }), {
fetchFn: (async () => {
disabledCalls += 1
return upstream()
}) as typeof fetch,
lookup,
})
const res = await request(off.adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${adminPermit()}`)
.send({ accessToken: 'tok' })
expect(res.status).toBe(404)
expect(res.body).toEqual({ error: 'disabled' })
expect(disabledCalls).toBe(0)
})
})
describe('grok-llm-proxy startup config', () => {
const base = {
GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
GROK_LLM_PROXY_CONTROL_API_URL:
'http://control-api-rpc-gateway.control-plane.svc.cluster.local:8090/api/v1',
GROK_LLM_PROXY_CONTROL_API_TOKEN: 'dev-grok-llm-proxy-token',
}
it('loads a complete environment', () => {
const loaded = loadConfig(base)
expect(loaded.controlApiBaseUrl).toBe(base.GROK_LLM_PROXY_CONTROL_API_URL)
expect(loaded.controlApiServiceToken).toBe('dev-grok-llm-proxy-token')
expect(loaded.controlApiServiceName).toBe('grok-llm-proxy')
})
// #731 — the envelope carries the contract-capped `request` plus the ticket,
// the hash and the deadline. A body limit equal to the contract cap refuses,
// as a 413, requests the contract itself accepts.
it('T-R2-6b-grok defaults the body limit to the contract request cap plus a 16 KiB envelope allowance', () => {
expect(loadConfig(base).maxBodyBytes).toBe(LIMITS.maxRequestBodyBytes + 16 * 1024)
})
// R11 — the allowance has one source, the contract, shared with control-api
// and mcp-host. A literal here could drift from theirs.
it('T-R11-grok takes the envelope allowance from the contract', () => {
expect(CONTRACT_ENVELOPE_ALLOWANCE_BYTES).toBe(16 * 1024)
expect(ENVELOPE_ALLOWANCE_BYTES).toBe(CONTRACT_ENVELOPE_ALLOWANCE_BYTES)
})
// R9-3 (L-3) — a lower override would answer 413 to requests the contract
// accepts, so it is refused at startup.
it('T-R9-3-grok refuses a body limit below the contract request cap plus the envelope allowance', () => {
const floor = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
expect(() => loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })).toThrow(
'GROK_LLM_PROXY_MAX_BODY_BYTES must be at least the contract request cap plus the envelope allowance'
)
expect(() =>
loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })
).toThrow(/GROK_LLM_PROXY_MAX_BODY_BYTES must be at least/)
// Witness: the floor itself and any larger value load.
expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(
floor
)
expect(
loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes
).toBe(floor + 1)
})
it('defaults the visual body limit to the contract visual ceiling, with no allowance on top', () => {
expect(LIMITS.maxVisualRequestBodyBytes).toBe(36_700_160)
expect(loadConfig(base).maxVisualBodyBytes).toBe(LIMITS.maxVisualRequestBodyBytes)
})
it('refuses a visual body limit below the contract visual ceiling', () => {
expect(() =>
loadConfig({
...base,
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes - 1),
})
).toThrow(
`GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at least ${LIMITS.maxVisualRequestBodyBytes}, the contract maxVisualRequestBodyBytes`
)
expect(() => loadConfig({ ...base, GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: '1024' })).toThrow(
/^GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at least 36700160,/
)
// Witness: the ceiling itself loads.
expect(
loadConfig({
...base,
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes),
}).maxVisualBodyBytes
).toBe(LIMITS.maxVisualRequestBodyBytes)
})
// The 2048Mi memory limit was measured with one visual stream at the contract
// ceiling. A larger visual limit would admit bodies that measurement never
// covered, so the proxy refuses to start with one.
it('refuses a visual body limit above the contract visual ceiling', () => {
expect(() =>
loadConfig({
...base,
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes + 1),
})
).toThrow(
`GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at most ${LIMITS.maxVisualRequestBodyBytes}, the contract maxVisualRequestBodyBytes`
)
expect(() =>
loadConfig({ ...base, GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(64 * 1024 * 1024) })
).toThrow(/^GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at most 36700160,/)
// Witness: the ceiling itself loads.
expect(
loadConfig({
...base,
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes),
}).maxVisualBodyBytes
).toBe(LIMITS.maxVisualRequestBodyBytes)
})
// With an equal budget every visual envelope would also fit the ordinary
// parser's cap, so the visual gate's separate admission and memory
// accounting would never engage.
it('refuses a visual body limit less than or equal to the ordinary body limit', () => {
const equal = LIMITS.maxVisualRequestBodyBytes
expect(() =>
loadConfig({
...base,
GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal),
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
})
).toThrow(
'GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be greater than GROK_LLM_PROXY_MAX_BODY_BYTES'
)
expect(() =>
loadConfig({
...base,
GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal + 1),
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
})
).toThrow(
'GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be greater than GROK_LLM_PROXY_MAX_BODY_BYTES'
)
// Liveness witness: a visual limit one byte above the ordinary limit loads.
// The visual limit is pinned to the contract ceiling, so the ordinary limit
// moves instead.
expect(
loadConfig({
...base,
GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal - 1),
GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
}).maxVisualBodyBytes
).toBe(equal)
})
it('T-R2-6c-grok does not refuse a request at the contract cap with a real ticket as payload_too_large', async () => {
const { runtimeApp } = createProxyApps(config({ maxBodyBytes: loadConfig(base).maxBodyBytes }))
const atCap = { pad: 'x'.repeat(LIMITS.maxRequestBodyBytes - '{"pad":""}'.length) }
expect(Buffer.byteLength(JSON.stringify(atCap), 'utf8')).toBe(LIMITS.maxRequestBodyBytes)
const res = await request(runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send({ executionTicket: ticket(), requestHash: 'a'.repeat(64), request: atCap })
expect(res.body.error).not.toBe('payload_too_large')
expect(res.status).not.toBe(413)
// Witness: the body parser accepted the envelope and the route answered.
expect(typeof res.body.error).toBe('string')
})
it.each([undefined, '', ' '])('fails at startup when the control-api URL is %j', value => {
expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
/GROK_LLM_PROXY_CONTROL_API_URL/
)
})
it.each(['not a url', 'ftp://control-api/api/v1', 'file:///etc/passwd'])(
'fails at startup when the control-api URL %j is not http(s)',
value => {
expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
/GROK_LLM_PROXY_CONTROL_API_URL/
)
}
)
it.each([undefined, '', ' '])(
'fails at startup when the control-api service token is %j',
value => {
expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_TOKEN: value })).toThrow(
/GROK_LLM_PROXY_CONTROL_API_TOKEN/
)
}
)
})
describe('grok-llm-proxy attempt telemetry', () => {
const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
// Keys that carry caller content. The attempt line is identifiers and counts;
// none of these may ever reach it. The four names this suite has always
// checked are joined by every leaf the service itself redacts, so a value the
// logger would scrub cannot be declared safe here merely because this list
// forgot it — `accessToken` and `attemptReceipt` both come back from the
// redeem fixture below.
const FORBIDDEN_LOG_KEYS = [
...new Set([
'body',
'request',
'executionTicket',
'arguments',
...REDACT_PATHS.map(
redacted =>
redacted
.replace(/\['([^']+)'\]/g, '.$1')
.split('.')
.pop() as string
).filter(key => key !== '*'),
]),
]
function completionBody(
providerAttemptId: string,
tamper?: (raw: Record<string, unknown>) => void
): Record<string, unknown> {
const raw: Record<string, unknown> = {
schemaVersion: 'grok-completion-request.v1',
requestId: `req-${providerAttemptId}`,
idempotencyKey: `idem-${providerAttemptId}`,
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'hello' }],
}
const parsed = parseGrokCompletionRequestV1(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
// Applied after hashing: the ticket stays bound to the untampered request,
// so the transport's parser rejects the body before any hash comparison.
tamper?.(raw)
const executionTicket = sign(
{
jti: '55555555-5555-4555-8555-555555555555',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId,
},
'grok-llm-proxy'
)
return { executionTicket, requestHash, request: raw }
}
function grantingClient(maxStreamDurationMs = 1_800_000): {
client: ControlApiClient
receipts: unknown[]
} {
const receipts: unknown[] = []
const client = {
async redeem(): Promise<RedeemAttemptSuccess> {
return {
accessToken: 'test-access-telemetry',
transport: {
protocolVersion: 'grok-subscription-transport.v1',
completionsOrigin: GROK_COMPLETIONS_ORIGIN,
catalogOrigin: GROK_CATALOG_ORIGIN,
operation: 'completion_stream',
servedModel: 'gpt-5.1',
maxStreamDurationMs,
},
expiryClass: 'short_lived',
attemptReceipt: 'c'.repeat(64),
}
},
async finalize(input: {
receipt: { outcome: FinalizeAttemptSuccess['outcome'] }
}): Promise<FinalizeAttemptSuccess> {
receipts.push(input.receipt)
return { providerAttemptId: 'att', outcome: input.receipt.outcome, duplicate: false }
},
} as unknown as ControlApiClient
return { client, receipts }
}
function upstream(textDeltas: number, calls: number): typeof fetch {
const frames: string[] = []
for (let index = 0; index < textDeltas; index += 1) {
frames.push(
`data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${index}` })}\n\n`
)
}
for (let index = 0; index < calls; index += 1) {
frames.push(
`data: ${JSON.stringify({
type: 'response.output_item.done',
item: { type: 'function_call', id: `call-${index}`, name: 'lookup', arguments: '{}' },
})}\n\n`
)
}
frames.push('data: {"type":"response.completed"}\n\n')
return (async () =>
new Response(frames.join(''), {
status: 200,
headers: { 'content-type': 'text/event-stream' },
})) as typeof fetch
}
// One SSE block larger than the transport's 1 MiB buffer and never terminated,
// so the buffer guard fires before any frame is delivered.
function unterminatedUpstream(): typeof fetch {
return (async () =>
new Response(`data: ${'x'.repeat(1_100_000)}`, {
status: 200,
headers: { 'content-type': 'text/event-stream' },
})) as typeof fetch
}
// One tool call whose `arguments` deltas add up past the retained budget,
// delivered one frame per read so the 1 MiB SSE buffer guard — which only
// bounds the unparsed tail between two `\n\n` boundaries — never fires and
// the argument budget is the guard under test.
const ARGUMENT_OVERRUN_CHUNK = 65_536
function oversizedArgumentsUpstream(): typeof fetch {
const chunks = MAX_TOOL_CALL_ARGUMENT_BYTES / ARGUMENT_OVERRUN_CHUNK + 1
const frames = [
`data: ${JSON.stringify({
type: 'response.output_item.added',
item: { type: 'function_call', id: 'call-args', name: 'lookup', arguments: '' },
})}\n\n`,
...Array.from(
{ length: chunks },
() =>
`data: ${JSON.stringify({
type: 'response.function_call_arguments.delta',
item_id: 'call-args',
delta: 'a'.repeat(ARGUMENT_OVERRUN_CHUNK),
})}\n\n`
),
'data: {"type":"response.completed"}\n\n',
]
const encoder = new TextEncoder()
return (async () => {
let index = 0
const body = new ReadableStream<Uint8Array>({
pull(controller) {
const next = frames[index]
index += 1
if (next === undefined) controller.close()
else controller.enqueue(encoder.encode(next))
},
})
return new Response(body, {
status: 200,
headers: { 'content-type': 'text/event-stream' },
})
}) as typeof fetch
}
// One call whose `arguments` are truncated JSON, closed by a completed response.
function malformedArgumentsUpstream(): typeof fetch {
const frames = [
`data: ${JSON.stringify({
type: 'response.output_item.done',
item: { type: 'function_call', id: 'call-bad', name: 'lookup', arguments: '{"q":' },
})}\n\n`,
'data: {"type":"response.completed"}\n\n',
]
return (async () =>
new Response(frames.join(''), {
status: 200,
headers: { 'content-type': 'text/event-stream' },
})) as typeof fetch
}
// The upstream's context-window refusal recorded on 2026-09-23 (R10): an
// HTTP 400 before any stream, with the marker inside the `error` string.
function contextOverflowUpstream(): typeof fetch {
return (async () =>
new Response(
JSON.stringify({
code: 'invalid-argument',
error:
"Failed to start sampling: [input_too_large] The prompt is too long for this model's context window (1107771 tokens > 500000 tokens)",
}),
{ status: 400, headers: { 'content-type': 'application/json' } }
)) as typeof fetch
}
function denyingClient(code: string): ControlApiClient {
return {
async redeem() {
throw new ControlApiClientError(code, 'control API request denied')
},
async finalize() {
throw new Error('finalize must not run after a denied redeem')
},
} as unknown as ControlApiClient
}
// Sends `textDeltas` frames, then never sends another byte. The body ignores
// the fetch signal, like a peer that stops writing without closing.
function stalledUpstream(textDeltas: number): typeof fetch {
return (async () => {
const encoder = new TextEncoder()
let sent = 0
const body = new ReadableStream<Uint8Array>(
{
pull(controller) {
if (sent < textDeltas) {
controller.enqueue(
encoder.encode(
`data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${sent}` })}\n\n`
)
)
sent += 1
return undefined
}
return new Promise<void>(() => undefined)
},
},
{ highWaterMark: 0 }
)
return new Response(body, {
status: 200,
headers: { 'content-type': 'text/event-stream' },
})
}) as typeof fetch
}
function failureCount(metricsText: string, code: string): number {
const line = metricsText
.split('\n')
.find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))
return line ? Number(line.split(' ').pop()) : 0
}
function timeoutCount(metricsText: string, kind: 'idle' | 'total'): number {
const line = metricsText
.split('\n')
.find(row => row.startsWith(`grok_proxy_upstream_timeouts_total{kind="${kind}"}`))
return line ? Number(line.split(' ').pop()) : 0
}
function keepaliveCount(text: string): number {
return text.split(': keepalive\n\n').length - 1
}
// Stays silent for `silentMs` after the fetch resolves, then streams one text
// delta and completes, like a model that reasons before its first token.
function silentThenCompletingUpstream(silentMs: number): typeof fetch {
return (async () =>
new Response(
new ReadableStream<Uint8Array>({
async start(controller) {
await new Promise(resolve => setTimeout(resolve, silentMs))
controller.enqueue(
new TextEncoder().encode(
'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
'data: {"type":"response.completed"}\n\n'
)
)
controller.close()
},
}),
{ status: 200, headers: { 'content-type': 'text/event-stream' } }
)) as typeof fetch
}
// Answers `status` with no SSE body after `delayMs`.
function slowFailingUpstream(delayMs: number, status: number): typeof fetch {
return (async () => {
await new Promise(resolve => setTimeout(resolve, delayMs))
return new Response('upstream failed', { status })
}) as typeof fetch
}
// Denies the redeem after `delayMs`, which is longer than the heartbeat
// interval the caller configures.
function slowDenyingClient(
code: string,
delayMs: number
): {
client: ControlApiClient
redeemCalls: () => number
} {
let calls = 0
const client = {
async redeem() {
calls += 1
await new Promise(resolve => setTimeout(resolve, delayMs))
throw new ControlApiClientError(code, 'control API request denied')
},
async finalize() {
throw new Error('finalize must not run after a denied redeem')
},
} as unknown as ControlApiClient
return { client, redeemCalls: () => calls }
}
async function run(options: {
providerAttemptId: string
textDeltas?: number
calls?: number
tamper?: (raw: Record<string, unknown>) => void
deniedCode?: string
fetchFn?: typeof fetch
maxStreamDurationMs?: number
configOverrides?: Partial<GrokLlmProxyConfig>
controlApiClient?: ControlApiClient
lookup?: (hostname: string) => Promise<Array<{ address: string; family: number }>>
}) {
const info = vi.spyOn(logger, 'info')
const warn = vi.spyOn(logger, 'warn')
const { client, receipts } = grantingClient(options.maxStreamDurationMs)
const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {
controlApiClient:
options.controlApiClient ??
(options.deniedCode ? denyingClient(options.deniedCode) : client),
fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),
lookup: options.lookup ?? lookup,
})
try {
const res = await request(apps.runtimeApp)
.post('/internal/runtime/v1/grok/completions')
.set('Authorization', `Bearer ${platformToken()}`)
.send(completionBody(options.providerAttemptId, options.tamper))
const metricsText = (await request(apps.probeApp).get('/metrics')).text
const logged = [...info.mock.calls, ...warn.mock.calls].map(
call => call[0] as unknown as Record<string, unknown>
)
const lines = logged.filter(entry => entry?.event === 'grok_proxy_attempt_finished')
const unreachable = logged.filter(
entry => entry?.event === 'grok_proxy_control_api_unreachable'
)
const causeCodeLines = logged.filter(entry => entry && 'causeCode' in entry)
return { res, receipts, lines, unreachable, causeCodeLines, metricsText }
} finally {
info.mockRestore()
warn.mockRestore()
}
}
/** Returns how many keys it inspected, so an empty payload cannot pass. */
function walkForbiddenKeys(value: unknown, trail: string): number {
if (Array.isArray(value)) {
return value.reduce<number>(
(total, item, index) => total + walkForbiddenKeys(item, `${trail}[${index}]`),
0
)
}
if (value === null || typeof value !== 'object') return 0
let visited = 0
for (const [key, nested] of Object.entries(value)) {
visited += 1
if (FORBIDDEN_LOG_KEYS.includes(key)) {
throw new Error(`log line carries the forbidden key ${trail}.${key}`)
}
visited += walkForbiddenKeys(nested, `${trail}.${key}`)
}
return visited
}
function expectNoForbiddenKeys(line: Record<string, unknown>): void {
// A flat check would clear `details` and `usage` without ever opening them.
const visited = walkForbiddenKeys(line, 'line')
// Liveness witness: a walk over an empty or non-object payload inspects
// nothing and would otherwise report the line as clean.
expect(visited).toBeGreaterThan(0)
}
// G1-1 (#720): an upstream 429 reaches the Host as a 429 rate_limited with
// the upstream's Retry-After, not as a retryable 503 provider_unavailable.
function rateLimitedUpstream(retryAfter?: string): typeof fetch {
return (async () =>
new Response('slow down', {
status: 429,
headers: retryAfter === undefined ? {} : { 'retry-after': retryAfter },
})) as typeof fetch
}
it('logs the error of an unmapped handler failure while it still answers 503 provider_unavailable', async () => {
const boom = new Error('boom')
const failingClient = {
async redeem(): Promise<RedeemAttemptSuccess> {
throw boom
},
async finalize(): Promise<FinalizeAttemptSuccess> {
throw new Error('finalize must not run after a failed redeem')
},
} as unknown as ControlApiClient
const { res, lines } = await run({
providerAttemptId: 'att-unmapped-error',
controlApiClient: failingClient,
})
// Witnesses: the request reached the handler's catch and was logged once.
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-unmapped-error',
outcome: 'failed',
code: 'provider_unavailable',
httpStatus: 503,
})
expect(lines[0]?.err).toBe(boom)
})
it('logs a mapped transport failure without an err entry', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-mapped-transport-error',
fetchFn: rateLimitedUpstream('7'),
})
// Witnesses: the failure reached the same catch and produced its attempt line.
expect(res.status).toBe(429)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-mapped-transport-error',
outcome: 'failed',
code: 'rate_limited',
})
expect('err' in (lines[0] ?? {})).toBe(false)
})
async function closedLoopbackPort(): Promise<number> {
const closed = createServer()
await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
const { port } = closed.address() as AddressInfo
await new Promise<void>(resolve => closed.close(() => resolve()))
return port
}
// Real undici fetch against a loopback port nothing listens on, so the
// failure is undici's own TypeError, whose cause message names the address.
function closedPortFetch(port: number, calls: { count: number }): typeof fetch {
return (async (_input: unknown, init?: RequestInit) => {
calls.count += 1
return fetch(`http://127.0.0.1:${port}/`, {
method: init?.method ?? 'GET',
signal: init?.signal ?? null,
})
}) as typeof fetch
}
// Serializes Error values with their message and cause, as the logger's err
// serializer does, so an error hidden in a log line cannot pass as clean.
function serializeWithErrors(value: unknown): string {
return JSON.stringify(value, (_key, nested: unknown) =>
nested instanceof Error
? { name: nested.name, message: nested.message, cause: nested.cause }
: nested
)
}
// Review R3-L1: a completion fetch nothing accepted reaches the handler as a
// mapped transport error, so the attempt line keeps the cause code and drops
// the err entry whose cause message names the upstream address.
it('(r3-l1a) logs an upstream connection refusal by cause code, without err or address', async () => {
const port = await closedLoopbackPort()
const calls = { count: 0 }
const { res, lines } = await run({
providerAttemptId: 'att-upstream-refused',
fetchFn: closedPortFetch(port, calls),
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
// Witnesses: the real fetch ran against the closed port, and the failure
// produced its attempt line.
expect(calls.count).toBe(1)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-upstream-refused',
outcome: 'failed',
code: 'provider_unavailable',
reason: 'upstream fetch failed',
causeCode: 'ECONNREFUSED',
deliveredAs: 'http_status',
httpStatus: 503,
})
expect('err' in lines[0]!).toBe(false)
expect(serializeWithErrors(lines[0])).not.toContain(`127.0.0.1:${port}`)
expectNoForbiddenKeys(lines[0]!)
})
// Review R3-L9: the admin route's refusal line carries only the code, so the
// catalog logs the cause code of a failed fetch before the 503.
it('(r3-l9a) logs a catalog connection refusal by cause code on both admin routes', async () => {
const port = await closedLoopbackPort()
for (const [route, operation] of [
['/internal/admin/v1/grok/models', 'catalog_list'],
['/internal/admin/v1/grok/test', 'connection_test'],
] as const) {
const calls = { count: 0 }
const warn = vi.spyOn(logger, 'warn')
const info = vi.spyOn(logger, 'info')
try {
const { adminApp } = createProxyApps(
config({ executionEnabled: true, maxBodyBytes: 65_536 }),
{ fetchFn: closedPortFetch(port, calls), lookup }
)
const res = await request(adminApp)
.post(route)
.set(
'Authorization',
`Bearer ${sign({ sub: 'admin-1', typ: 'grok-admin-permit', operation }, 'grok-llm-proxy-admin')}`
)
.send({ accessToken: 'tok' })
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
// Witness: the real fetch ran against the closed port.
expect(calls.count).toBe(1)
const logged = [...warn.mock.calls, ...info.mock.calls].map(
call => call[0] as unknown as Record<string, unknown>
)
expect(logged.filter(entry => entry?.event === 'grok_catalog_upstream')).toEqual([
{ event: 'grok_catalog_upstream', causeCode: 'ECONNREFUSED' },
])
expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
{ event: 'grok_proxy_denied', code: 'provider_unavailable' },
])
expect(serializeWithErrors(logged)).not.toContain(`127.0.0.1:${port}`)
} finally {
warn.mockRestore()
info.mockRestore()
}
}
})
it('(r3-l9b) answers a catalog body that is not JSON with a mapped 503 and its own log line', async () => {
let calls = 0
const warn = vi.spyOn(logger, 'warn')
try {
const { adminApp } = createProxyApps(
config({ executionEnabled: true, maxBodyBytes: 65_536 }),
{
fetchFn: (async () => {
calls += 1
return new Response('<html>maintenance</html>', {
status: 200,
headers: { 'content-type': 'text/html' },
})
}) as typeof fetch,
lookup,
}
)
const res = await request(adminApp)
.post('/internal/admin/v1/grok/models')
.set('Authorization', `Bearer ${adminPermit()}`)
.send({ accessToken: 'tok' })
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
expect(calls).toBe(1)
const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
expect(logged.filter(entry => entry?.event === 'grok_catalog_upstream')).toEqual([
{ event: 'grok_catalog_upstream', reason: 'invalid_json' },
])
expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
{ event: 'grok_proxy_denied', code: 'provider_unavailable' },
])
} finally {
warn.mockRestore()
}
})
// Review R4-L1: a DNS failure carries its code on the lookup error itself,
// not in `cause`, and its message names the upstream host. The attempt line
// and both admin routes log it by cause code, with no err entry and no host.
// These real .invalid lookups depend on the resolver honoring NXDOMAIN.
const INVALID_UPSTREAM_HOST = 'grok-r4-l1.invalid'
function invalidHostLookup(calls: { count: number }) {
return async () => {
calls.count += 1
const { lookup: dnsLookup } = await import('node:dns/promises')
const records = await dnsLookup(INVALID_UPSTREAM_HOST, { all: true })
return records.map(record => ({ address: record.address, family: record.family }))
}
}
function countingFetch(calls: { count: number }): typeof fetch {
return (async () => {
calls.count += 1
throw new Error('fetch must not run after a failed lookup')
}) as typeof fetch
}
it('(r4-l1a) logs a DNS lookup failure by cause code, without err or host', async () => {
const lookups = { count: 0 }
const fetches = { count: 0 }
const { res, lines } = await run({
providerAttemptId: 'att-upstream-dns',
fetchFn: countingFetch(fetches),
lookup: invalidHostLookup(lookups),
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
// Witnesses: the real lookup ran once, failed before any fetch, and the
// failure produced its attempt line.
expect(lookups.count).toBe(1)
expect(fetches.count).toBe(0)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-upstream-dns',
outcome: 'failed',
code: 'provider_unavailable',
reason: 'upstream fetch failed',
deliveredAs: 'http_status',
httpStatus: 503,
})
expect(String(lines[0]!.causeCode)).toMatch(/^(ENOTFOUND|EAI_AGAIN)$/)
expect('err' in lines[0]!).toBe(false)
expect(serializeWithErrors(lines[0])).not.toContain(INVALID_UPSTREAM_HOST)
expectNoForbiddenKeys(lines[0]!)
}, 15_000)
// Both admin routes look up serially: 30 s bounds two 15 s lookup budgets.
it('(r4-l1b) logs a catalog DNS lookup failure by cause code on both admin routes', async () => {
for (const [route, operation] of [
['/internal/admin/v1/grok/models', 'catalog_list'],
['/internal/admin/v1/grok/test', 'connection_test'],
] as const) {
const lookups = { count: 0 }
const fetches = { count: 0 }
const warn = vi.spyOn(logger, 'warn')
const info = vi.spyOn(logger, 'info')
const error = vi.spyOn(logger, 'error')
try {
const { adminApp } = createProxyApps(
config({ executionEnabled: true, maxBodyBytes: 65_536 }),
{ fetchFn: countingFetch(fetches), lookup: invalidHostLookup(lookups) }
)
const res = await request(adminApp)
.post(route)
.set(
'Authorization',
`Bearer ${sign({ sub: 'admin-1', typ: 'grok-admin-permit', operation }, 'grok-llm-proxy-admin')}`
)
.send({ accessToken: 'tok' })
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
// Witnesses: the real lookup ran once and failed before any fetch.
expect(lookups.count).toBe(1)
expect(fetches.count).toBe(0)
const logged = [...warn.mock.calls, ...info.mock.calls, ...error.mock.calls].map(
call => call[0] as unknown as Record<string, unknown>
)
const upstream = logged.filter(entry => entry?.event === 'grok_catalog_upstream')
expect(upstream).toHaveLength(1)
expect(Object.keys(upstream[0]!).sort()).toEqual(['causeCode', 'event'])
expect(String(upstream[0]!.causeCode)).toMatch(/^(ENOTFOUND|EAI_AGAIN)$/)
expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
{ event: 'grok_proxy_denied', code: 'provider_unavailable' },
])
expect(serializeWithErrors(logged)).not.toContain(INVALID_UPSTREAM_HOST)
} finally {
warn.mockRestore()
info.mockRestore()
error.mockRestore()
}
}
}, 30_000)
it('logs a mapped control-api failure without an err entry', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-mapped-control-api-error',
deniedCode: 'ticket_expired',
})
expect(res.status).toBeGreaterThanOrEqual(400)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-mapped-control-api-error',
outcome: 'failed',
code: 'ticket_expired',
})
expect('err' in (lines[0] ?? {})).toBe(false)
})
it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(
'normalizes real transport upstream 503 claiming %s to provider_unavailable',
async code => {
const fetchFn = vi
.fn<typeof fetch>()
.mockResolvedValue(
Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })
)
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-spoofed-local',
fetchFn,
})
expect(fetchFn).toHaveBeenCalledTimes(1)
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
code: 'provider_unavailable',
deliveredAs: 'http_status',
httpStatus: 503,
})
expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
expect(failureCount(metricsText, 'request_limit')).toBe(0)
}
)
it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-rate-http',
fetchFn: rateLimitedUpstream('7'),
})
expect(res.status).toBe(429)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.headers['retry-after']).toBe('7')
expect(res.body).toEqual({ error: 'rate_limited' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-rate-http',
outcome: 'failed',
code: 'rate_limited',
details: { retryAfterSeconds: 7 },
deliveredAs: 'http_status',
httpStatus: 429,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'rate_limited')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
it('(g1-1b) answers 429 with no Retry-After header when the upstream value is invalid', async () => {
const { res } = await run({
providerAttemptId: 'att-rate-bad-header',
fetchFn: rateLimitedUpstream('3601'),
})
// Witness: the 429 path ran.
expect(res.status).toBe(429)
expect(res.body).toEqual({ error: 'rate_limited' })
expect(res.headers['retry-after']).toBeUndefined()
})
// G1-3 (#720): an upstream refusal the same request would get again is a
// non-retryable 422 upstream_rejected, not a retryable 503. 502 stays the
// gateway's "nothing answered" signal (review R1-H2).
it('(g1-3a) answers 422 upstream_rejected with the upstream status', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-rejected-http',
fetchFn: (async () => new Response('not found', { status: 404 })) as typeof fetch,
})
expect(res.status).toBe(422)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'upstream_rejected', upstreamStatus: 404 })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-rejected-http',
outcome: 'failed',
code: 'upstream_rejected',
details: { upstreamStatus: 404 },
deliveredAs: 'http_status',
httpStatus: 422,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'upstream_rejected')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
it('(g1-3c) carries the upstream status on the SSE error frame after a keepalive', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-rejected-sse',
fetchFn: slowFailingUpstream(60, 402),
configOverrides: { heartbeatIntervalMs: 20 },
})
expect(res.status).toBe(200)
// Witness: a keepalive went out first, so only the frame can carry it.
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
expect(
res.text.endsWith(
'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'
)
).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
code: 'upstream_rejected',
details: { upstreamStatus: 402 },
deliveredAs: 'sse_error',
})
})
it('(g1-1d) delivers an upstream 429 after a keepalive as an SSE rate_limited frame', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-rate-sse',
fetchFn: (async () => {
await new Promise(resolve => setTimeout(resolve, 60))
return new Response('slow down', { status: 429, headers: { 'retry-after': '7' } })
}) as typeof fetch,
configOverrides: { heartbeatIntervalMs: 20 },
})
expect(res.status).toBe(200)
// Witness: a keepalive went out first, so the 429 can only travel as a frame.
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
// The frame carries the code alone: no Retry-After, no upstream status.
expect(res.text.endsWith('data: {"type":"error","code":"rate_limited"}\n\n')).toBe(true)
expect(res.headers['retry-after']).toBeUndefined()
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
code: 'rate_limited',
details: { retryAfterSeconds: 7 },
deliveredAs: 'sse_error',
})
})
// G1-4 (#720): a redeem whose connection nothing accepted (control-api or
// its gateway restarting) is reported as the control plane being down.
it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {
const closed = createServer()
await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
const { port } = closed.address() as AddressInfo
await new Promise<void>(resolve => closed.close(() => resolve()))
const upstreamFetch = vi.fn(upstream(1, 0))
const { res, lines, unreachable, causeCodeLines, metricsText } = await run({
providerAttemptId: 'att-control-down',
fetchFn: upstreamFetch as unknown as typeof fetch,
controlApiClient: new ControlApiClient({
baseUrl: `http://127.0.0.1:${port}/api/v1`,
serviceName: 'grok-llm-proxy',
serviceToken: 'dev-grok-llm-proxy-token',
}),
})
expect(res.status).toBe(503)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'control_plane_unavailable' })
// Redeem failed, so the upstream was never called.
expect(upstreamFetch).not.toHaveBeenCalled()
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-control-down',
outcome: 'failed',
code: 'control_plane_unavailable',
causeCode: 'ECONNREFUSED',
deliveredAs: 'http_status',
httpStatus: 503,
})
expectNoForbiddenKeys(lines[0]!)
// Review R1-M2: the hop line counts the failure; the cause code is on the
// attempt line only, so one failure never reads as two.
expect(unreachable).toHaveLength(1)
expect('causeCode' in unreachable[0]!).toBe(false)
expect(causeCodeLines).toEqual([lines[0]])
expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
it('(a) answers 422 and logs one attempt line when 257 calls arrive before any text', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-limit-http',
calls: 257,
})
expect(res.status).toBe(422)
// The staged SSE headers are replaced by a JSON response.
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.headers['cache-control']).toBeUndefined()
expect(res.body).toEqual({ error: 'tool_call_limit_exceeded' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-limit-http',
outcome: 'failed',
code: 'tool_call_limit_exceeded',
reason: 'tool calls exceed 256',
details: { limit: 256, observed: 257 },
deliveredAs: 'http_status',
httpStatus: 422,
toolCalls: 0,
textChunks: 0,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(1)
})
// A call count within `maxToolCalls` says nothing about the size of each
// call. This refusal has to reach the Host as a 422 with its own code: a 503
// would be classified as an overload and retried with the same oversized
// response.
it('(a1) answers 422 when one call’s arguments exceed the retained budget', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-args-http',
fetchFn: oversizedArgumentsUpstream(),
})
expect(res.status).toBe(422)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'tool_call_arguments_exceeded' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-args-http',
outcome: 'failed',
code: 'tool_call_arguments_exceeded',
details: {
limit: MAX_TOOL_CALL_ARGUMENT_BYTES,
observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_OVERRUN_CHUNK,
},
deliveredAs: 'http_status',
httpStatus: 422,
toolCalls: 0,
textChunks: 0,
})
// The refusal text names the bound, never the arguments that tripped it.
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'tool_call_arguments_exceeded')).toBe(1)
})
// Arguments that are not a JSON object are invalid model output. The refusal
// has to reach the Host as a 422 with its own code and metric label: the 503
// default would be classified as an overload and retried.
it('(a1b) answers 422 invalid_tool_arguments when a call’s arguments are not a JSON object', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-bad-args-http',
fetchFn: malformedArgumentsUpstream(),
})
expect(res.status).toBe(422)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'invalid_tool_arguments' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-bad-args-http',
outcome: 'failed',
code: 'invalid_tool_arguments',
deliveredAs: 'http_status',
httpStatus: 422,
toolCalls: 0,
textChunks: 0,
})
expectNoForbiddenKeys(lines[0]!)
// The attempt line names the refusal, never the arguments that caused it.
expect(JSON.stringify(lines[0])).not.toContain('{\\"q\\":')
expect(failureCount(metricsText, 'invalid_tool_arguments')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
// R10 (M1): the upstream context overflow reaches the Host as a 400 with its
// own code and metric label. The 503 default would be classified as an
// overload and retried with the identical conversation.
it('(a1c) T-R10-2 answers 400 context_length_exceeded for the upstream context overflow', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-context-http',
fetchFn: contextOverflowUpstream(),
})
expect(res.status).toBe(400)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'context_length_exceeded' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-context-http',
outcome: 'failed',
code: 'context_length_exceeded',
deliveredAs: 'http_status',
httpStatus: 400,
toolCalls: 0,
textChunks: 0,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'context_length_exceeded')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
it('(a2) sends an SSE error frame when text was already streamed', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-limit-sse',
textDeltas: 1,
calls: 257,
})
expect(res.status).toBe(200)
expect(res.text).toContain('data: {"type":"text","text":"t0"}')
expect(res.text).toContain('data: {"type":"error","code":"tool_call_limit_exceeded"}')
expect(res.text).not.toContain('"type":"tool_call"')
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
code: 'tool_call_limit_exceeded',
deliveredAs: 'sse_error',
textChunks: 1,
toolCalls: 0,
})
expect('httpStatus' in lines[0]!).toBe(false)
expectNoForbiddenKeys(lines[0]!)
})
it('(b) logs the counted frames of a successful attempt', async () => {
const { res, lines, metricsText } = await run({
providerAttemptId: 'att-success',
calls: 3,
})
expect(res.status).toBe(200)
expect(res.text).toContain('"type":"done","outcome":"success"')
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-success',
outcome: 'success',
deliveredAs: 'sse_done',
toolCalls: 3,
textChunks: 0,
})
expectNoForbiddenKeys(lines[0]!)
// Witness: the attempt was counted, so the absent failure sample is meaningful.
expect(metricsText).toMatch(
/grok_llm_proxy_attempts_total\{outcome="success",operation="completion_stream"\} 1/
)
expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(0)
})
it('(c) serves a post-staging invalid request as JSON without the parse message', async () => {
const secretField = 'sk-live-0123456789abcdef'
const { res, lines } = await run({
providerAttemptId: 'att-invalid',
tamper: raw => {
;(raw.messages as Record<string, unknown>[])[0]![secretField] = 'x'
},
})
// This error is raised inside the transport, after the SSE headers were
// staged. The route's four earlier rejections run before that staging and
// already answer JSON, so only this one proves the repair.
expect(res.status).toBe(400)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.headers['cache-control']).toBeUndefined()
expect(res.body).toEqual({ error: 'invalid_request' })
// Witness: the attempt line is still emitted for the rejected request.
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-invalid',
outcome: 'failed',
code: 'invalid_request',
deliveredAs: 'http_status',
httpStatus: 400,
})
// The parser's message names caller-supplied fields, so it is withheld.
expect('reason' in lines[0]!).toBe(false)
expect(JSON.stringify(lines[0])).not.toContain(secretField)
expectNoForbiddenKeys(lines[0]!)
})
it('(t1) answers 504 stream_duration_exceeded when the total cap fires before any frame', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-total-http',
fetchFn: stalledUpstream(0),
maxStreamDurationMs: 100,
})
expect(res.status).toBe(504)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'stream_duration_exceeded' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-total-http',
outcome: 'failed',
code: 'stream_duration_exceeded',
reason: 'upstream stream exceeded maxStreamDurationMs',
details: { limitMs: 100 },
deliveredAs: 'http_status',
httpStatus: 504,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'stream_duration_exceeded')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
expect(timeoutCount(metricsText, 'total')).toBe(1)
expect(timeoutCount(metricsText, 'idle')).toBe(0)
})
it('(t2) sends a stream_duration_exceeded SSE frame when text was already streamed', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-total-sse',
fetchFn: stalledUpstream(1),
maxStreamDurationMs: 100,
})
expect(res.status).toBe(200)
expect(res.text).toContain('data: {"type":"text","text":"t0"}')
expect(res.text).toContain('data: {"type":"error","code":"stream_duration_exceeded"}')
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
code: 'stream_duration_exceeded',
deliveredAs: 'sse_error',
textChunks: 1,
})
expect('httpStatus' in lines[0]!).toBe(false)
})
it('(t3) answers 503 provider_unavailable and counts an idle timeout on a silent upstream', async () => {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-idle-http',
fetchFn: stalledUpstream(0),
configOverrides: { upstreamIdleTimeoutMs: 50 },
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'provider_unavailable' })
expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-idle-http',
outcome: 'failed',
code: 'provider_unavailable',
reason: 'upstream stream idle timeout',
details: { idleTimeoutMs: 50 },
deliveredAs: 'http_status',
httpStatus: 503,
})
expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
expect(timeoutCount(metricsText, 'idle')).toBe(1)
expect(timeoutCount(metricsText, 'total')).toBe(0)
})
it('(d) labels the failure metric with a known transport code', async () => {
const { res, lines, metricsText } = await run({
providerAttemptId: 'att-buffer',
fetchFn: unterminatedUpstream(),
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'sse_buffer_exceeded' })
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
code: 'sse_buffer_exceeded',
httpStatus: 503,
})
expect(failureCount(metricsText, 'sse_buffer_exceeded')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
it('(e) labels an unknown control-api code as other and keeps the raw code in the log', async () => {
const rawCode = 'k8s says: pod foo not found'
const { res, lines, metricsText } = await run({
providerAttemptId: 'att-unknown-code',
deniedCode: rawCode,
})
expect(res.status).toBe(503)
// Witness: the attempt failed and was logged with the code control-api sent.
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({ outcome: 'failed', code: rawCode, httpStatus: 503 })
expect(failureCount(metricsText, 'other')).toBe(1)
expect(metricsText).not.toContain(rawCode)
})
// G1-5 (#720): the rpc gateway answers a redeem it could not deliver with
// JSON `control_plane_unavailable`; the proxy passes it through with its own
// status and metric label.
it('(g1-5) passes a control_plane_unavailable redeem denial through as 503 with its own label', async () => {
const { res, lines, metricsText } = await run({
providerAttemptId: 'att-cp-json',
deniedCode: 'control_plane_unavailable',
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'control_plane_unavailable' })
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
code: 'control_plane_unavailable',
httpStatus: 503,
})
expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
expect(failureCount(metricsText, 'other')).toBe(0)
})
// `ATTEMPT_ERROR_STATUS` is an object literal, so an inherited name resolves
// to a function or an object instead of undefined. A plain index read would
// hand that value to `res.status()`, which throws ERR_HTTP_INVALID_STATUS_CODE
// inside the request IIFE's catch — an unhandled rejection that takes every
// other in-flight stream down with the process.
it.each(['constructor', '__proto__', 'toString', 'hasOwnProperty'])(
'(f) answers 503 for the inherited control-api code %j instead of crashing',
async code => {
const { res, lines, metricsText } = await run({
providerAttemptId: `att-proto-${code}`,
deniedCode: code,
})
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: code })
// Witness: the attempt was reached and logged, so the status above is the
// mapped refusal and not a connection that never produced a response.
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({ outcome: 'failed', code, httpStatus: 503 })
expect(failureCount(metricsText, 'other')).toBe(1)
expect(failureCount(metricsText, code)).toBe(0)
}
)
it('(rl1) logs the request-limit reason and labels its failure metric request_limit', async () => {
const acquire = vi
.spyOn(streamGate, 'acquire')
.mockRejectedValueOnce(
new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')
)
try {
const { res, receipts, lines, metricsText } = await run({
providerAttemptId: 'att-queue-full',
})
// Witness: the refusal came from the stream gate this test replaced.
expect(acquire).toHaveBeenCalledTimes(1)
expect(res.status).toBe(503)
expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })
// The gate refused before the redeem, so there is no receipt to finalize.
expect(receipts).toEqual([])
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-queue-full',
outcome: 'failed',
code: 'proxy_capacity_exceeded',
reason: 'stream queue is full',
deliveredAs: 'http_status',
httpStatus: 503,
})
expectNoForbiddenKeys(lines[0]!)
expect(failureCount(metricsText, 'request_limit')).toBe(1)
expect(failureCount(metricsText, 'provider_unavailable')).toBe(0)
} finally {
acquire.mockRestore()
}
})
it('(hb1) keeps a silent upstream attempt open with SSE comments and counts them', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-heartbeat',
fetchFn: silentThenCompletingUpstream(100),
configOverrides: { heartbeatIntervalMs: 20 },
})
expect(res.status).toBe(200)
const firstData = res.text.indexOf('data:')
expect(firstData).toBeGreaterThan(0)
expect(keepaliveCount(res.text.slice(0, firstData))).toBeGreaterThanOrEqual(2)
expect(res.text).toContain('data: {"type":"text","text":"t0"}')
// Nothing is written after the done frame.
expect(res.text.endsWith('data: {"type":"done","outcome":"success"}\n\n')).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
providerAttemptId: 'att-heartbeat',
outcome: 'success',
deliveredAs: 'sse_done',
heartbeats: keepaliveCount(res.text),
})
})
it('(hb2) sends no keepalive before the redeem succeeds, so a denial keeps its HTTP status', async () => {
const denied = slowDenyingClient('no_grant', 60)
const { res, lines } = await run({
providerAttemptId: 'att-heartbeat-denied',
controlApiClient: denied.client,
configOverrides: { heartbeatIntervalMs: 20 },
})
// Witness: the redeem ran and outlasted three heartbeat intervals.
expect(denied.redeemCalls()).toBe(1)
expect(res.status).toBe(403)
expect(res.headers['content-type']).toMatch(/^application\/json/)
expect(res.body).toEqual({ error: 'no_grant' })
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
code: 'no_grant',
deliveredAs: 'http_status',
httpStatus: 403,
heartbeats: 0,
})
})
it('(hb3) delivers an upstream failure after a keepalive as an SSE error frame', async () => {
const { res, lines } = await run({
providerAttemptId: 'att-heartbeat-failed',
fetchFn: slowFailingUpstream(60, 500),
configOverrides: { heartbeatIntervalMs: 20 },
})
expect(res.status).toBe(200)
expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)
expect(lines).toHaveLength(1)
expect(lines[0]).toMatchObject({
outcome: 'failed',
code: 'provider_unavailable',
deliveredAs: 'sse_error',
heartbeats: keepaliveCount(res.text),
})
expect('httpStatus' in lines[0]!).toBe(false)
})
// Node drops a write on a closed response without an error, so a heartbeat
// the server never stops is invisible on the wire: it is an interval that
// outlives its attempt. The test tracks the timers themselves.
it('(hb4) clears every heartbeat timer once the attempt ends and keeps serving', async () => {
const uncaught: unknown[] = []
const onUncaught = (err: unknown): void => {
uncaught.push(err)
}
const realSetInterval = globalThis.setInterval
const heartbeatTimers: unknown[] = []
const setSpy = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
handler: () => void,
ms?: number
) => {
const timer = realSetInterval(handler, ms)
// 5 ms is the interval this test configures; nothing else uses it.
if (ms === 5) heartbeatTimers.push(timer)
return timer
}) as unknown as typeof setInterval)
const clearSpy = vi.spyOn(globalThis, 'clearInterval')
process.on('uncaughtException', onUncaught)
try {
const first = await run({
providerAttemptId: 'att-heartbeat-end',
fetchFn: silentThenCompletingUpstream(30),
configOverrides: { heartbeatIntervalMs: 5 },
})
// Witness: the heartbeat was running while the upstream was silent.
expect(keepaliveCount(first.res.text)).toBeGreaterThanOrEqual(1)
await new Promise(resolve => setTimeout(resolve, 50))
expect(uncaught).toEqual([])
const second = await run({
providerAttemptId: 'att-heartbeat-next',
textDeltas: 1,
configOverrides: { heartbeatIntervalMs: 5 },
})
expect(second.res.status).toBe(200)
expect(second.res.text).toContain('data: {"type":"done","outcome":"success"}')
// Witness: each attempt started its own heartbeat.
expect(heartbeatTimers).toHaveLength(2)
const cleared = clearSpy.mock.calls.map(call => call[0])
for (const timer of heartbeatTimers) expect(cleared).toContain(timer)
} finally {
process.off('uncaughtException', onUncaught)
setSpy.mockRestore()
clearSpy.mockRestore()
}
})
})
/**
* #739 D1-bis — the stream-gate wait also ends when the execution ticket dies.
* A request still queued at the ticket's `exp` could only be redeemed into a
* certain `ticket_expired`, so it is answered as a capacity refusal instead,
* with no redeem. The margin is zero: a request whose ticket is still alive
* when a slot frees is served exactly as before.
*/
describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
/** Captured before any test fakes the clock, so waits stay on real time. */
const realSetTimeout = globalThis.setTimeout
const realSleep = (ms: number) => new Promise<void>(resolve => realSetTimeout(resolve, ms))
const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
const COMPLETIONS = '/internal/runtime/v1/grok/completions'
async function until(condition: () => boolean, what: string): Promise<void> {
const deadline = performance.now() + 5_000
while (!condition()) {
if (performance.now() > deadline) throw new Error(`${what} did not happen within 5 s`)
await realSleep(10)
}
}
function withinReal<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
return Promise.race([promise, realSleep(ms).then(() => undefined)])
}
/** An envelope whose ticket expires exactly `ticketLifeMs` from the (fake) now. */
function envelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {
const raw: Record<string, unknown> = {
schemaVersion: 'grok-completion-request.v1',
requestId: `req-${providerAttemptId}`,
idempotencyKey: `idem-${providerAttemptId}`,
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'hello' }],
}
const parsed = parseGrokCompletionRequestV1(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
const executionTicket = jwt.sign(
{
jti: '77777777-7777-4777-8777-777777777777',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId,
exp: (Date.now() + ticketLifeMs) / 1000,
},
privateKey,
{ algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
)
return { executionTicket, requestHash, request: raw }
}
function countingClient(redeemAnswer: 'granted' | 'ticket_expired'): {
client: ControlApiClient
redeems: () => number
} {
let redeems = 0
const client = {
async redeem(): Promise<RedeemAttemptSuccess> {
redeems += 1
if (redeemAnswer === 'ticket_expired') {
throw new ControlApiClientError('ticket_expired', 'control API request denied')
}
return {
accessToken: 'test-access-ticket-life',
transport: {
protocolVersion: 'grok-subscription-transport.v1',
completionsOrigin: GROK_COMPLETIONS_ORIGIN,
catalogOrigin: GROK_CATALOG_ORIGIN,
operation: 'completion_stream',
servedModel: 'gpt-5.1',
maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'd'.repeat(64),
}
},
async finalize(input: {
receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
}): Promise<FinalizeAttemptSuccess> {
return {
providerAttemptId: input.receipt.providerAttemptId,
outcome: input.receipt.outcome,
duplicate: false,
}
},
} as unknown as ControlApiClient
return { client, redeems: () => redeems }
}
const upstream = (async () =>
new Response(
'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
'data: {"type":"response.completed"}\n\n',
{ status: 200, headers: { 'content-type': 'text/event-stream' } }
)) as typeof fetch
function failureCount(metricsText: string, code: string): number {
const line = metricsText
.split('\n')
.find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))
return line ? Number(line.split(' ').pop()) : 0
}
/** Fakes the clock on a whole second, so a ticket's `exp` lands on an exact instant. */
function fakeClock(): void {
vi.useFakeTimers({
now: Math.ceil(Date.now() / 1000) * 1000,
toFake: ['setTimeout', 'clearTimeout', 'Date'],
})
}
/**
* Frees the held slots and lets any waiter still polling the shared gate
* finish on the fake clock, so the module gate is empty for the next test.
*/
async function releaseAndDrain(slots: Array<() => void>): Promise<void> {
for (const release of slots.splice(0)) release()
await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)
vi.useRealTimers()
}
async function saturate(): Promise<Array<() => void>> {
const slots: Array<() => void> = []
for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {
slots.push(await streamGate.acquire())
}
return slots
}
it('T-AC-5-grok answers 503 provider_unavailable at the ticket expiry, without a redeem, while the gate stays saturated', async () => {
fakeClock()
const warn = vi.spyOn(logger, 'warn')
let slots: Array<() => void> = []
let acquire: ReturnType<typeof vi.spyOn> | undefined
try {
slots = await saturate()
// Pass-through spy: it only counts the handler's calls into the gate.
const spy = vi.spyOn(streamGate, 'acquire')
acquire = spy
const control = countingClient('ticket_expired')
const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
controlApiClient: control.client,
fetchFn: upstream,
lookup,
})
const ticketLifeMs = 20_000
const reply = request(apps.runtimeApp)
.post(COMPLETIONS)
.set('Authorization', `Bearer ${platformToken()}`)
.send(envelope('att-ticket-life', ticketLifeMs))
.then(res => res)
await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')
await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)
// Witness: the request waited for the ticket's whole life, it was not
// refused on arrival.
expect(await withinReal(reply, 100)).toBeUndefined()
await vi.advanceTimersByTimeAsync(1)
const res = await withinReal(reply, 2_000)
expect(res?.status).toBe(503)
expect(res?.body).toEqual({ error: 'provider_unavailable' })
// A slot that frees after the ticket died is never used to redeem it.
slots.pop()?.()
await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)
expect(control.redeems()).toBe(0)
const refusals = warn.mock.calls
.map(call => call[0] as unknown as Record<string, unknown>)
.filter(entry => entry?.event === 'grok_proxy_admission_refused')
expect(refusals).toEqual([
{
event: 'grok_proxy_admission_refused',
reason: 'ticket_life',
providerAttemptId: 'att-ticket-life',
hostRef: 'research-host',
},
])
const metricsText = (await request(apps.probeApp).get('/metrics')).text
expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
expect(failureCount(metricsText, 'request_limit')).toBe(0)
} finally {
acquire?.mockRestore()
await releaseAndDrain(slots)
warn.mockRestore()
}
}, 30_000)
it('T-PAR-TK-grok serves a request whose ticket is still alive when the gate frees at +50 s of a 60 s ticket', async () => {
fakeClock()
let slots: Array<() => void> = []
let acquire: ReturnType<typeof vi.spyOn> | undefined
try {
slots = await saturate()
const spy = vi.spyOn(streamGate, 'acquire')
acquire = spy
const control = countingClient('granted')
const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
controlApiClient: control.client,
fetchFn: upstream,
lookup,
})
const reply = request(apps.runtimeApp)
.post(COMPLETIONS)
.set('Authorization', `Bearer ${platformToken()}`)
.send(envelope('att-ticket-alive', 60_000))
.then(res => res)
await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')
await vi.advanceTimersByTimeAsync(50_000)
expect(await withinReal(reply, 100)).toBeUndefined()
slots.pop()?.()
// One stream-gate poll interval.
await vi.advanceTimersByTimeAsync(10)
const res = await withinReal(reply, 2_000)
expect(res?.status).toBe(200)
expect(res?.text).toContain('data: {"type":"text","text":"t0"}')
expect(res?.text).toContain('data: {"type":"done","outcome":"success"}')
expect(control.redeems()).toBe(1)
} finally {
acquire?.mockRestore()
await releaseAndDrain(slots)
}
}, 30_000)
/** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */
function visualEnvelope(
providerAttemptId: string,
ticketLifeMs: number
): Record<string, unknown> {
const raw: Record<string, unknown> = {
schemaVersion: 'grok-completion-request.v2',
requestId: `req-${providerAttemptId}`,
idempotencyKey: `idem-${providerAttemptId}`,
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'x'.repeat(80_000) }],
}
const parsed = parseGrokCompletionRequest(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
const executionTicket = jwt.sign(
{
jti: '78787878-7878-4787-8787-787878787878',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId,
exp: (Date.now() + ticketLifeMs) / 1000,
},
privateKey,
{ algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
)
return { executionTicket, requestHash, request: raw }
}
// R17-2 on the visual path: the visual gate is taken before the body is
// parsed, so the ticket is read only once the slot frees. A ticket that died
// during that wait reaches the ticket gate expired, and the answer is the
// retryable ticket_expired, not the stream gate's ticket-life refusal.
it('T-R17-2-visual-grok answers ticket_expired when the ticket died while the body waited at the visual gate', async () => {
fakeClock()
const warn = vi.spyOn(logger, 'warn')
let held: (() => void) | undefined
try {
held = await visualStreamGate.acquire()
const control = countingClient('granted')
const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
controlApiClient: control.client,
fetchFn: upstream,
lookup,
})
const ticketLifeMs = 20_000
const body = visualEnvelope('att-visual-expired', ticketLifeMs)
expect(Buffer.byteLength(JSON.stringify(body))).toBeGreaterThan(65_536)
const reply = request(apps.runtimeApp)
.post(COMPLETIONS)
.set('Authorization', `Bearer ${platformToken()}`)
.send(body)
.then(res => res)
await until(
() => visualStreamGate.snapshot().queued === 1,
'the body queueing at the visual gate'
)
await vi.advanceTimersByTimeAsync(ticketLifeMs)
// Witness: the body was still waiting for the slot at the ticket's expiry.
expect(await withinReal(reply, 100)).toBeUndefined()
held()
held = undefined
// One stream-gate poll interval.
await vi.advanceTimersByTimeAsync(10)
const res = await withinReal(reply, 2_000)
expect(res?.status).toBe(403)
expect(res?.body).toEqual({ error: 'ticket_expired' })
expect(control.redeems()).toBe(0)
const events = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
expect(events.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
{ event: 'grok_proxy_denied', code: 'ticket_expired' },
])
expect(events.filter(entry => entry?.event === 'grok_proxy_admission_refused')).toEqual([])
// The ticket refusal released the visual slot the body was parsed under.
expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
expect(streamGate.snapshot()).toEqual({ running: 0, queued: 0 })
} finally {
held?.()
vi.useRealTimers()
warn.mockRestore()
}
}, 30_000)
// The visual gate's own wait is the admission clock stamped at arrival. A
// body that reaches the gate some ms after arrival and waits that clock out
// is refused as the gate's own queue wait, `visual_gate`: the same answer as
// a body that reached the gate in the arrival millisecond.
it.each([0, 5])(
'T-VG-QW-grok answers 503 visual_gate when a body enqueued %i ms after arrival waits out the admission clock',
async lagMs => {
fakeClock()
const warn = vi.spyOn(logger, 'warn')
const held: Array<() => void> = []
let acquire: { mockRestore: () => void } | undefined
try {
for (let i = 0; i < VISUAL_STREAM_LIMITS.maxConcurrentStreams; i += 1) {
held.push(await visualStreamGate.acquire())
}
const enqueue = visualStreamGate.acquire.bind(visualStreamGate)
// Moves the clock `lagMs` between the arrival stamp and the enqueue,
// then calls the real gate.
const spy = vi
.spyOn(visualStreamGate, 'acquire')
.mockImplementation((signal, deadlineAt, queueWaitStartedAt) => {
vi.setSystemTime(Date.now() + lagMs)
return enqueue(signal, deadlineAt, queueWaitStartedAt)
})
acquire = spy
const control = countingClient('granted')
const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
controlApiClient: control.client,
fetchFn: upstream,
lookup,
})
const arrivedAt = Date.now()
const reply = request(apps.runtimeApp)
.post(COMPLETIONS)
.set('Authorization', `Bearer ${platformToken()}`)
.send(visualEnvelope(`att-visual-wait-${lagMs}`, 60_000))
.then(res => res)
await until(
() => spy.mock.calls.length === 1 && visualStreamGate.snapshot().queued === 1,
'the body queueing at the visual gate'
)
// Witness: the gate was handed the admission clock and the arrival
// instant, and it enqueued the body `lagMs` after arrival.
expect(spy.mock.calls[0]?.slice(1)).toEqual([
arrivedAt + STREAM_LIMITS.maxQueueWaitMs,
arrivedAt,
])
expect(Date.now()).toBe(arrivedAt + lagMs)
await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - lagMs - 1)
// Witness: still queued one millisecond before the admission clock
// runs out, so the body was not refused early.
expect(await withinReal(reply, 100)).toBeUndefined()
expect(visualStreamGate.snapshot().queued).toBe(1)
await vi.advanceTimersByTimeAsync(1)
const res = await withinReal(reply, 2_000)
expect(Date.now()).toBe(arrivedAt + STREAM_LIMITS.maxQueueWaitMs)
expect(res?.status).toBe(503)
expect(res?.body).toEqual({ error: 'visual_gate' })
expect(control.redeems()).toBe(0)
const refusals = warn.mock.calls
.map(call => call[0] as unknown as Record<string, unknown>)
.filter(entry => entry?.event === 'grok_proxy_admission_refused')
expect(refusals).toEqual([
{
event: 'grok_proxy_admission_refused',
reason: 'visual_gate',
code: 'visual_gate',
detail: 'stream queue wait exceeded',
},
])
expect(visualStreamGate.snapshot()).toEqual({ running: held.length, queued: 0 })
} finally {
acquire?.mockRestore()
for (const release of held.splice(0)) release()
vi.useRealTimers()
warn.mockRestore()
}
},
30_000
)
})
/**
* #739 D2 — a body's budget reservation covers the phase in which several
* copies of it are alive: reading, parsing, hashing and forwarding. That phase
* ends when the upstream fetch resolves, because the whole request body has
* been written by then. The reservation is released there instead of when the
* SSE stream closes; the response's `close` event still releases it on every
* path that never reaches the upstream.
*/
describe('grok-llm-proxy body budget release on upstream acceptance (#739 D2)', () => {
const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
const COMPLETIONS = '/internal/runtime/v1/grok/completions'
const sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))
async function until(condition: () => boolean, what: string): Promise<void> {
const deadline = performance.now() + 10_000
while (!condition()) {
if (performance.now() > deadline) throw new Error(`${what} did not happen within 10 s`)
await sleep(10)
}
}
/** A runtime envelope with one user message of `contentChars` and a 60 s ticket. */
function completionBody(
contentChars: number,
providerAttemptId: string,
executionTicket?: string
): string {
const raw = {
schemaVersion: 'grok-completion-request.v1',
requestId: `req-${providerAttemptId}`,
idempotencyKey: `idem-${providerAttemptId}`,
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'x'.repeat(contentChars) }],
}
const parsed = parseGrokCompletionRequestV1(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
return JSON.stringify({
executionTicket:
executionTicket ??
sign(
{
jti: '88888888-8888-4888-8888-888888888888',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId,
},
'grok-llm-proxy'
),
requestHash,
request: raw,
})
}
function controlClient(redeemAnswer: 'granted' | 'ticket_expired'): {
client: ControlApiClient
redeems: () => number
} {
let redeems = 0
const client = {
async redeem(): Promise<RedeemAttemptSuccess> {
redeems += 1
if (redeemAnswer === 'ticket_expired') {
throw new ControlApiClientError('ticket_expired', 'control API request denied')
}
return {
accessToken: 'test-access-body-release',
transport: {
protocolVersion: 'grok-subscription-transport.v1',
completionsOrigin: GROK_COMPLETIONS_ORIGIN,
catalogOrigin: GROK_CATALOG_ORIGIN,
operation: 'completion_stream',
servedModel: 'gpt-5.1',
maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'e'.repeat(64),
}
},
async finalize(input: {
receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
}): Promise<FinalizeAttemptSuccess> {
return {
providerAttemptId: input.receipt.providerAttemptId,
outcome: input.receipt.outcome,
duplicate: false,
}
},
} as unknown as ControlApiClient
return { client, redeems: () => redeems }
}
type UpstreamProbe = {
fetchFn: typeof fetch
calls: () => number
/** True once the signal the proxy gave the upstream fetch has aborted. */
aborted: () => boolean
}
function upstreamProbe(answer: (signal: AbortSignal) => Promise<Response>): UpstreamProbe {
let calls = 0
let seen: AbortSignal | undefined
const fetchFn = (async (_url: unknown, init?: RequestInit) => {
calls += 1
const signal = init?.signal
if (!signal) throw new Error('the upstream fetch carried no abort signal')
seen = signal
return answer(signal)
}) as typeof fetch
return { fetchFn, calls: () => calls, aborted: () => seen?.aborted === true }
}
/** Answers headers and one text delta, then stays open until the fetch aborts. */
function openStreamUpstream(): UpstreamProbe {
return upstreamProbe(
async signal =>
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
new TextEncoder().encode(
'data: {"type":"response.output_text.delta","delta":"t0"}\n\n'
)
)
signal.addEventListener('abort', () => controller.error(signal.reason), {
once: true,
})
},
}),
{ status: 200, headers: { 'content-type': 'text/event-stream' } }
)
)
}
/** Never answers: the fetch stays pending until it aborts. */
function pendingUpstream(): UpstreamProbe {
return upstreamProbe(
signal =>
new Promise<Response>((_resolve, reject) => {
signal.addEventListener('abort', () => reject(signal.reason), { once: true })
})
)
}
/**
* The runtime app on a real listener, so an SSE stream can be observed while
* it is still open. The body budget is private to `createProxyApps`; a
* pass-through spy on `BodyBudget.prototype.acquire` records the instance.
*/
async function listeningProxy(client: ControlApiClient, fetchFn: typeof fetch) {
const acquire = vi.spyOn(BodyBudget.prototype, 'acquire')
const apps = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }), {
controlApiClient: client,
fetchFn,
lookup,
})
let closes = 0
apps.runtime.on('request', (_req, res) => {
res.once('close', () => {
closes += 1
})
})
await new Promise<void>(resolve => apps.runtime.listen(0, '127.0.0.1', () => resolve()))
const { port } = apps.runtime.address() as AddressInfo
return {
port,
/** Declared sizes the body budget was asked for, in call order. */
reservations: () => acquire.mock.calls.map(call => call[0]),
budget: (): BodyBudget => {
const instance = acquire.mock.contexts[0]
if (!(instance instanceof BodyBudget)) throw new Error('no body reached the body budget')
return instance
},
/** Responses whose `close` event fired. */
closes: () => closes,
close: async () => {
acquire.mockRestore()
await apps.close()
},
}
}
/** POSTs `payload` with a declared length and records the reply as it arrives. */
function open(port: number, payload: string) {
let status: number | undefined
let received = ''
let ended = false
const errors: Error[] = []
const req = httpRequest(
{
host: '127.0.0.1',
port,
method: 'POST',
path: COMPLETIONS,
agent: false,
headers: {
'content-type': 'application/json',
authorization: `Bearer ${platformToken()}`,
'content-length': Buffer.byteLength(payload),
},
},
res => {
status = res.statusCode
res.setEncoding('utf8')
res.on('data', (chunk: string) => {
received += chunk
})
res.on('end', () => {
ended = true
})
}
)
// Recorded, not ignored: a test that destroys the request expects one.
req.on('error', err => errors.push(err))
req.end(payload)
return {
status: () => status,
received: () => received,
ended: () => ended,
errors: () => errors,
destroy: () => req.destroy(),
}
}
it('T-BR-1-grok releases the reservation of a near-cap body once the upstream accepted it, while the SSE stream is still open', async () => {
const control = controlClient('granted')
const upstream = openStreamUpstream()
const proxy = await listeningProxy(control.client, upstream.fetchFn)
const payload = completionBody(Math.floor(LIMITS.maxRequestBodyBytes * 0.99), 'att-br-1')
const bytes = Buffer.byteLength(payload)
// Fixture check: the body is near the 8 MiB cap and still admitted by it.
expect(bytes).toBeGreaterThan(LIMITS.maxRequestBodyBytes * 0.99)
expect(bytes).toBeLessThanOrEqual(DEFAULT_MAX_BODY_BYTES)
const client = open(proxy.port, payload)
try {
// Witness: the first frame reached the client, so the upstream accepted
// the request and the stream is live.
await until(
() => client.received().includes('data: {"type":"text","text":"t0"}'),
'the first SSE frame'
)
expect(client.status()).toBe(200)
expect(client.ended()).toBe(false)
expect(upstream.calls()).toBe(1)
expect(control.redeems()).toBe(1)
expect(proxy.reservations()).toEqual([bytes])
expect(proxy.budget().inFlightBytes).toBe(0)
} finally {
client.destroy()
await proxy.close()
}
}, 30_000)
it('T-BR-2-grok keeps the reservation while the upstream fetch is pending and releases it when the client leaves', async () => {
const control = controlClient('granted')
const upstream = pendingUpstream()
const proxy = await listeningProxy(control.client, upstream.fetchFn)
const payload = completionBody(64, 'att-br-2')
const bytes = Buffer.byteLength(payload)
const client = open(proxy.port, payload)
try {
// Witness: the attempt was redeemed and its upstream fetch started.
await until(() => upstream.calls() === 1, 'the upstream fetch')
expect(control.redeems()).toBe(1)
expect(proxy.reservations()).toEqual([bytes])
// Not released early: the fetch has not resolved, so the body may still
// be being written.
expect(proxy.budget().inFlightBytes).toBe(bytes)
client.destroy()
await until(() => upstream.aborted(), 'the client abort reaching the upstream fetch')
await until(() => proxy.budget().inFlightBytes === 0, 'the reservation release')
expect(client.status()).toBeUndefined()
} finally {
client.destroy()
await proxy.close()
}
}, 30_000)
it('T-BR-3a-grok releases the reservation of a request refused 403 ticket_invalid when its response closes', async () => {
const control = controlClient('granted')
const upstream = openStreamUpstream()
const proxy = await listeningProxy(control.client, upstream.fetchFn)
const payload = completionBody(64, 'att-br-3a', 'not-a-signed-ticket')
const client = open(proxy.port, payload)
try {
await until(() => client.ended(), 'the refusal')
expect(client.status()).toBe(403)
expect(JSON.parse(client.received())).toEqual({ error: 'ticket_invalid' })
await until(() => proxy.closes() === 1, "the response's close event")
expect(client.errors()).toEqual([])
expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
expect(control.redeems()).toBe(0)
expect(proxy.budget().inFlightBytes).toBe(0)
} finally {
await proxy.close()
}
}, 30_000)
it('T-BR-3b-grok releases the reservation of a request refused 503 by a full stream gate when its response closes', async () => {
const control = controlClient('granted')
const upstream = openStreamUpstream()
const proxy = await listeningProxy(control.client, upstream.fetchFn)
const slots: Array<() => void> = []
const queueAbort = new AbortController()
const waiters: Array<Promise<() => void>> = []
try {
for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {
slots.push(await streamGate.acquire())
}
for (let i = 0; i < STREAM_LIMITS.maxQueuedRequests; i += 1) {
waiters.push(streamGate.acquire(queueAbort.signal))
}
// Fixture check: the gate has no snapshot, so a probe shows the queue is full.
await expect(streamGate.acquire()).rejects.toMatchObject({
name: 'RequestLimitError',
message: 'stream queue is full',
})
const payload = completionBody(64, 'att-br-3b')
const client = open(proxy.port, payload)
await until(() => client.ended(), 'the refusal')
expect(client.status()).toBe(503)
expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })
await until(() => proxy.closes() === 1, "the response's close event")
expect(client.errors()).toEqual([])
expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
expect(control.redeems()).toBe(0)
expect(proxy.budget().inFlightBytes).toBe(0)
} finally {
queueAbort.abort()
await Promise.allSettled(waiters)
for (const release of slots.splice(0)) release()
await proxy.close()
}
}, 30_000)
it('T-BR-3c-grok releases the reservation of a request whose redeem failed when its response closes', async () => {
const control = controlClient('ticket_expired')
const upstream = openStreamUpstream()
const proxy = await listeningProxy(control.client, upstream.fetchFn)
const payload = completionBody(64, 'att-br-3c')
const client = open(proxy.port, payload)
try {
await until(() => client.ended(), 'the refusal')
expect(client.status()).toBe(403)
expect(JSON.parse(client.received())).toEqual({ error: 'ticket_expired' })
await until(() => proxy.closes() === 1, "the response's close event")
expect(client.errors()).toEqual([])
// Witness: the path reached the redeem and stopped before the upstream.
expect(control.redeems()).toBe(1)
expect(upstream.calls()).toBe(0)
expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
expect(proxy.budget().inFlightBytes).toBe(0)
} finally {
await proxy.close()
}
}, 30_000)
})
/**
* #739 D6 — on SIGTERM `main.ts` awaits `servers.close()`. The Deployment's
* termination grace period (the stream cap plus 60 s) is only useful if that
* close waits for an open SSE stream instead of cutting it.
*/
describe('grok-llm-proxy graceful drain on shutdown (#739 D6)', () => {
const sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))
async function until(condition: () => boolean, what: string): Promise<void> {
const deadline = performance.now() + 10_000
while (!condition()) {
if (performance.now() > deadline) throw new Error(`${what} did not happen within 10 s`)
await sleep(10)
}
}
function completionBody(): string {
const raw = {
schemaVersion: 'grok-completion-request.v1',
requestId: 'req-att-drain',
idempotencyKey: 'idem-att-drain',
provider: 'grok-subscription',
model: 'gpt-5.1',
messages: [{ role: 'user', content: 'hello' }],
}
const parsed = parseGrokCompletionRequestV1(raw)
if (!parsed.ok) throw new Error(parsed.message)
const requestHash = hashGrokCompletionRequest(parsed.value)
const executionTicket = sign(
{
jti: '99999999-9999-4999-8999-999999999999',
typ: 'grok-execution-ticket',
hostRef: 'research-host',
model: 'gpt-5.1',
requestHash,
providerAttemptId: 'att-drain',
},
'grok-llm-proxy'
)
return JSON.stringify({ executionTicket, requestHash, request: raw })
}
const client = {
async redeem(): Promise<RedeemAttemptSuccess> {
return {
accessToken: 'test-access-drain',
transport: {
protocolVersion: 'grok-subscription-transport.v1',
completionsOrigin: GROK_COMPLETIONS_ORIGIN,
catalogOrigin: GROK_CATALOG_ORIGIN,
operation: 'completion_stream',
servedModel: 'gpt-5.1',
maxStreamDurationMs: 1_800_000,
},
expiryClass: 'short_lived',
attemptReceipt: 'f'.repeat(64),
}
},
async finalize(input: {
receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
}): Promise<FinalizeAttemptSuccess> {
return {
providerAttemptId: input.receipt.providerAttemptId,
outcome: input.receipt.outcome,
duplicate: false,
}
},
} as unknown as ControlApiClient
it('T-DR-1-grok close() does not resolve while an SSE stream is open and resolves after the stream ended', async () => {
const encoder = new TextEncoder()
let finishUpstream: (() => void) | undefined
const fetchFn = (async () =>
new Response(
new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
encoder.encode('data: {"type":"response.output_text.delta","delta":"t0"}\n\n')
)
finishUpstream = () => {
controller.enqueue(encoder.encode('data: {"type":"response.completed"}\n\n'))
controller.close()
}
},
}),
{ status: 200, headers: { 'content-type': 'text/event-stream' } }
)) as typeof fetch
const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
controlApiClient: client,
fetchFn,
lookup: async () => [{ address: '1.2.3.4', family: 4 }],
})
const order: string[] = []
apps.runtime.on('request', (_req, res) => {
res.once('finish', () => order.push('response finished'))
})
await new Promise<void>(resolve => apps.runtime.listen(0, '127.0.0.1', () => resolve()))
const { port } = apps.runtime.address() as AddressInfo
const payload = completionBody()
let received = ''
let ended = false
const req = httpRequest(
{
host: '127.0.0.1',
port,
method: 'POST',
path: '/internal/runtime/v1/grok/completions',
agent: false,
headers: {
'content-type': 'application/json',
authorization: `Bearer ${platformToken()}`,
'content-length': Buffer.byteLength(payload),
},
},
res => {
res.setEncoding('utf8')
res.on('data', (chunk: string) => {
received += chunk
})
res.on('end', () => {
ended = true
})
}
)
const errors: Error[] = []
req.on('error', err => errors.push(err))
req.end(payload)
let closing: Promise<void> | undefined
try {
await until(
() => received.includes('data: {"type":"text","text":"t0"}'),
'the first SSE frame'
)
closing = apps.close().then(() => {
order.push('closed')
})
await sleep(300)
// Witness: the stream is still open, and close() is still waiting on it.
expect(ended).toBe(false)
expect(order).toEqual([])
if (!finishUpstream) throw new Error('the upstream stream never started')
finishUpstream()
await until(() => order.includes('closed'), 'close() resolving')
expect(order).toEqual(['response finished', 'closed'])
await until(() => ended, 'the client seeing the end of the stream')
expect(received).toContain('data: {"type":"done","outcome":"success"}')
expect(errors).toEqual([])
} finally {
req.destroy()
if (!closing) await apps.close()
else await closing
}
}, 30_000)
})