Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
126 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import { generateKeyPairSync } from 'node:crypto'
import { createServer, request as httpRequest } from 'node:http'
import type { AddressInfo } from 'node:net'
import request from 'supertest'
import {
  ENVELOPE_ALLOWANCE_BYTES as CONTRACT_ENVELOPE_ALLOWANCE_BYTES,
  LIMITS,
  hashGrokCompletionRequest,
  parseGrokCompletionRequest,
  parseGrokCompletionRequestV1,
} from '@clerum/grok-provider-attempt-contract'
import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'
import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'
import { type GrokLlmProxyConfig, loadConfig } from '../src/config.js'
import {
  ControlApiClient,
  ControlApiClientError,
  type FinalizeAttemptSuccess,
  type RedeemAttemptSuccess,
} from '../src/controlApiClient.js'
import { GrokTransportError, MAX_TOOL_CALL_ARGUMENT_BYTES } from '../src/grokTransport.js'
import { REDACT_PATHS, logger } from '../src/logger.js'
import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
import {
  BodyBudget,
  DEFAULT_MAX_BODY_BYTES,
  ENVELOPE_ALLOWANCE_BYTES,
  RequestLimitError,
  STREAM_LIMITS,
  VISUAL_STREAM_LIMITS,
  streamGate,
  visualStreamGate,
} from '../src/requestLimits.js'
import { createProxyApps } from '../src/server.js'

const { privateKey, publicKey } = generateKeyPairSync('rsa', {
  modulusLength: 2048,
  publicKeyEncoding: { type: 'spki', format: 'pem' },
  privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
})

function config(overrides: Partial<GrokLlmProxyConfig> = {}): GrokLlmProxyConfig {
  return {
    runtimePort: 8080,
    adminPort: 8081,
    probePort: 9090,
    maxBodyBytes: 1024,
    maxVisualBodyBytes: LIMITS.maxVisualRequestBodyBytes,
    maxStreamDurationMs: 1_800_000,
    maxDeadlineMs: 1_800_000,
    upstreamIdleTimeoutMs: 600_000,
    heartbeatIntervalMs: 15_000,
    jwtIssuer: 'control-api',
    jwtPublicKey: publicKey,
    executionEnabled: true,
    controlApiBaseUrl: '',
    controlApiServiceName: 'grok-llm-proxy',
    controlApiServiceToken: '',
    ...overrides,
  }
}

function sign(payload: Record<string, unknown>, audience: string): string {
  return jwt.sign(payload, privateKey, {
    algorithm: 'RS256',
    issuer: 'control-api',
    audience,
    expiresIn: 60,
  })
}

function platformToken(overrides: Record<string, unknown> = {}): string {
  return sign(
    {
      sub: 'default/research-host',
      hostRefs: ['research-host'],
      workflowControlScopes: ['llm:grok:execute'],
      scope: 'workflow:approval:request',
      ...overrides,
    },
    'workflow-approvals'
  )
}

function ticket(): string {
  return sign(
    {
      jti: '11111111-1111-4111-8111-111111111111',
      typ: 'grok-execution-ticket',
      hostRef: 'research-host',
      model: 'gpt-5.1',
      requestHash: 'a'.repeat(64),
      providerAttemptId: 'att-1',
    },
    'grok-llm-proxy'
  )
}

function adminPermit(): string {
  return sign(
    { sub: 'admin-1', typ: 'grok-admin-permit', operation: 'catalog_list' },
    'grok-llm-proxy-admin'
  )
}

describe('grok-llm-proxy security surface', () => {
  it('exposes only the frozen runtime, admin, and probe routes', async () => {
    const { runtimeApp, adminApp, probeApp } = createProxyApps(config())
    expect((await request(probeApp).get('/healthz')).status).toBe(200)
    expect((await request(probeApp).get('/readyz')).status).toBe(200)
    const metrics = await request(probeApp).get('/metrics')
    expect(metrics.status).toBe(200)
    expect(metrics.text).not.toMatch(/account|refresh|accessToken/i)
    expect((await request(runtimeApp).get('/internal/runtime/v1/grok/completions')).status).toBe(
      404
    )
    expect((await request(adminApp).get('/internal/admin/v1/grok/models')).status).toBe(404)
  })

  it('denies an admin permit on the runtime listener and a runtime ticket on admin', async () => {
    const { runtimeApp, adminApp } = createProxyApps(config())
    const runtime = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${adminPermit()}`)
      .send({ executionTicket: ticket(), requestHash: 'a'.repeat(64), request: {} })
    expect(runtime.status).toBe(403)
    expect(runtime.body.error).toBe('insufficient_scope')

    const admin = await request(adminApp)
      .post('/internal/admin/v1/grok/models')
      .set('Authorization', `Bearer ${ticket()}`)
      .send({})
    expect(admin.status).toBe(403)
    expect(admin.body.error).toBe('insufficient_scope')
  })

  it('rejects unknown fields, invalid deadlines, and incorrect content types', async () => {
    const { runtimeApp } = createProxyApps(config())
    const unknown = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
        extra: true,
      })
    expect(unknown.body.error).toBe('unknown_field')

    const deadline = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
        deadlineMs: 1_800_001,
      })
    expect(deadline.body.error).toBe('invalid_request')

    const ctype = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .set('Content-Type', 'text/plain')
      .send('not-json')
    expect(ctype.status).toBe(415)
  })

  it('rejects an oversized body before invoking the completion path', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: 32 }))
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: { pad: 'x'.repeat(200) },
      })
    expect(res.status).toBe(413)
  })

  it('(g1-2) rate limits the completion endpoint with a JSON rate_limited body', async () => {
    const { runtimeApp } = createProxyApps(config())
    const completion = () => request(runtimeApp).post('/internal/runtime/v1/grok/completions')
    // Within the window every request reaches the platform JWT gate: the witness
    // that the 61st rejection comes from the limiter and not from that gate.
    for (let i = 0; i < 60; i += 1) {
      const accepted = await completion().send({})
      expect(accepted.status).toBe(401)
    }
    const limited = await completion().send({})
    expect(limited.status).toBe(429)
    // G1-2 (#720): the Host reads the JSON error code; a text body would fall
    // back to provider_unavailable ("Model Overloaded").
    expect(limited.headers['content-type']).toMatch(/^application\/json/)
    expect(limited.body).toEqual({ error: 'rate_limited' })
    // The library's own headers stay: Retry-After and the draft-7 pair.
    expect(limited.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)
    expect(limited.headers['ratelimit-policy']).toBe('60;w=60')
  })

  it('reserves the visual transport budget for an authenticated ~30 MiB V2 body', async () => {
    const { runtimeApp, adminApp } = createProxyApps(
      config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES })
    )
    // Deliberately invalid ticket: this test checks parser admission and the
    // unchanged ticket gate, without redeeming or contacting any model. 30 MiB
    // is past the Codex 24 MiB visual ceiling and inside the Grok 35 MiB one.
    const payload = {
      executionTicket: 'invalid-ticket',
      requestHash: 'a'.repeat(64),
      request: {
        schemaVersion: 'grok-completion-request.v2',
        messages: [
          {
            role: 'user',
            contentParts: [{ type: 'image', data: 'A'.repeat(30 * 1024 * 1024) }],
          },
        ],
      },
    }
    expect(Buffer.byteLength(JSON.stringify(payload))).toBeGreaterThan(24 * 1024 * 1024)
    const acquire = vi.spyOn(visualStreamGate, 'acquire')
    const admitted = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send(payload)
    expect(admitted.status).toBe(403)
    expect(admitted.body.error).toBe('ticket_invalid')
    // Witness: the body went through the visual gate, not the ordinary parser.
    expect(acquire).toHaveBeenCalledTimes(1)
    expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })

    // R9-M-B: without a platform JWT the body is never read, so the caller
    // gets 401 instead of a parser's 413. The 401 is written before the upload
    // ends; a keep-alive client has the unread rest discarded.
    const anonymous = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Connection', 'keep-alive')
      .send(payload)
    expect(anonymous.status).toBe(401)
    const noScope = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Connection', 'keep-alive')
      .set('Authorization', `Bearer ${platformToken({ workflowControlScopes: [] })}`)
      .send(payload)
    expect(noScope.status).toBe(401)
    const v1 = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        ...payload,
        request: { ...payload.request, schemaVersion: 'grok-completion-request.v1' },
      })
    expect(v1.status).toBe(413)
    expect(v1.body.error).toBe('payload_too_large')
    const admin = await request(adminApp)
      .post('/internal/admin/v1/grok/models')
      .set('Authorization', `Bearer ${adminPermit()}`)
      .send(payload)
    expect(admin.status).toBe(413)
    expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
    acquire.mockRestore()
  }, 30_000)

  // r10 Y1: the ticket is a signed JWT of a few hundred bytes. The non-image
  // ceiling alone lets an ~8 MiB ticket reach jwt.verify, so the schema bounds
  // it at the envelope allowance.
  it('T-Y1-grok bounds the execution ticket at the envelope allowance before verifying it', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
    const send = (executionTicket: string, body: Record<string, unknown> = {}) =>
      request(runtimeApp)
        .post('/internal/runtime/v1/grok/completions')
        .set('Authorization', `Bearer ${platformToken()}`)
        .send({ executionTicket, requestHash: 'a'.repeat(64), request: body })

    // Witness: a ticket exactly at the bound reaches the verifier.
    const atBound = await send('x'.repeat(ENVELOPE_ALLOWANCE_BYTES))
    expect(atBound.status).toBe(403)
    expect(atBound.body.error).toBe('ticket_invalid')

    const overBound = await send('x'.repeat(ENVELOPE_ALLOWANCE_BYTES + 1))
    expect(overBound.status).toBe(400)
    expect(overBound.body.error).toBe('invalid_request')

    // A V2 body pushed past the ordinary cap by its ticket and one small image
    // takes the visual gate, is refused at the schema, and frees its slot.
    const acquire = vi.spyOn(visualStreamGate, 'acquire')
    const visual = await send('x'.repeat(8_000_000), {
      schemaVersion: 'grok-completion-request.v2',
      messages: [
        { role: 'user', contentParts: [{ type: 'image', data: 'A'.repeat(1024 * 1024) }] },
      ],
    })
    expect(visual.status).toBe(400)
    expect(visual.body.error).toBe('invalid_request')
    expect(acquire).toHaveBeenCalledTimes(1)
    expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
    acquire.mockRestore()
  }, 30_000)

  it('refuses a declared length past the visual ceiling with 413 before reading it', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
    const pad = 'A'.repeat(LIMITS.maxVisualRequestBodyBytes)
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .set('Connection', 'keep-alive')
      .send({
        executionTicket: 'invalid-ticket',
        requestHash: 'a'.repeat(64),
        request: { schemaVersion: 'grok-completion-request.v2', pad },
      })
    expect(res.status).toBe(413)
    expect(res.body.error).toBe('payload_too_large')
    expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
  }, 30_000)

  it('answers a transport payload_too_large with HTTP 413 before any SSE byte', async () => {
    const raw = {
      schemaVersion: 'grok-completion-request.v2',
      requestId: 'req-payload-too-large',
      idempotencyKey: 'idem-payload-too-large',
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'hello' }],
    }
    const parsed = parseGrokCompletionRequest(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    const executionTicket = sign(
      {
        jti: '12121212-1212-4121-8121-121212121212',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId: 'att-payload-too-large',
      },
      'grok-llm-proxy'
    )
    let streamCalls = 0
    const { runtimeApp, probeApp } = createProxyApps(config({ maxBodyBytes: 65_536 }), {
      streamCompletion: async () => {
        streamCalls += 1
        throw new GrokTransportError(
          'payload_too_large',
          'request exceeds maxVisualRequestBodyBytes'
        )
      },
    })
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({ executionTicket, requestHash, request: raw })
    // Witness: the request reached the transport.
    expect(streamCalls).toBe(1)
    expect(res.status).toBe(413)
    expect(res.body).toEqual({ error: 'payload_too_large' })
    const metricsText = (await request(probeApp).get('/metrics')).text
    expect(metricsText).toMatch(
      /^grok_proxy_attempt_failures_total\{code="payload_too_large"\} 1$/m
    )
  })

  it('does not let a V2 declaration raise the non-image budget to 35 MiB', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
    // One byte of text past the non-image ceiling plus its envelope allowance,
    // far under the 35 MiB visual ceiling the V2 declaration opens.
    const pad = 'x'.repeat(LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES + 1)
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        executionTicket: 'invalid-ticket',
        requestHash: 'a'.repeat(64),
        request: { schemaVersion: 'grok-completion-request.v2', pad },
      })
    expect(res.status).toBe(413)
    expect(res.body.error).toBe('payload_too_large')
    // Witness: the same body with an image-sized pad inside contentParts
    // instead of text passes the size checks and stops at the ticket gate.
    const imageSized = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({
        executionTicket: 'invalid-ticket',
        requestHash: 'a'.repeat(64),
        request: {
          schemaVersion: 'grok-completion-request.v2',
          messages: [{ role: 'user', contentParts: [{ type: 'image', data: pad }] }],
        },
      })
    expect(imageSized.status).toBe(403)
    expect(imageSized.body.error).toBe('ticket_invalid')
  }, 30_000)

  // Review R3-L3: V2 text repeated in content and in its text parts counts
  // once on the non-image budget, as the same text does in V1.
  it('R3-L3 counts V2 text repeated in content and its text parts once', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }))
    const completion = (content: string, textPart: string) => ({
      executionTicket: 'invalid-ticket',
      requestHash: 'a'.repeat(64),
      request: {
        schemaVersion: 'grok-completion-request.v2',
        messages: [
          {
            role: 'user',
            content,
            contentParts: [
              { type: 'text', text: textPart },
              { type: 'image', mimeType: 'image/png', data: 'iVBORw0KGgo=' },
            ],
          },
        ],
      },
    })
    const post = (content: string, textPart: string) =>
      request(runtimeApp)
        .post('/internal/runtime/v1/grok/completions')
        .set('Authorization', `Bearer ${platformToken()}`)
        .send(completion(content, textPart))
    // The share the proxy measures: no ticket, no image data, and the text
    // part blanked because it repeats content.
    const { executionTicket: _ticket, ...empty } = completion('', '')
    empty.request.messages[0].contentParts[1] = { type: 'image', mimeType: 'image/png', data: '' }
    const budget = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
    const exact = budget - Buffer.byteLength(JSON.stringify(empty), 'utf8')

    // A 4.5 MiB prompt beside an image passes the size checks and stops at the
    // ticket gate.
    const text = 'x'.repeat(4.5 * 1024 * 1024)
    const repeated = await post(text, text)
    expect(repeated.status).toBe(403)
    expect(repeated.body).toEqual({ error: 'ticket_invalid' })

    // Exact boundary: the largest text reaches the ticket gate, one more byte
    // is refused 413.
    const atLimit = await post('x'.repeat(exact), 'x'.repeat(exact))
    expect(atLimit.status).toBe(403)
    expect(atLimit.body).toEqual({ error: 'ticket_invalid' })
    const overLimit = await post('x'.repeat(exact + 1), 'x'.repeat(exact + 1))
    expect(overLimit.status).toBe(413)
    expect(overLimit.body).toEqual({ error: 'payload_too_large' })

    // Parts that do not repeat content keep both copies and are refused.
    const mismatched = await post(`${text.slice(1)}y`, text)
    expect(mismatched.status).toBe(413)
    expect(mismatched.body).toEqual({ error: 'payload_too_large' })
  }, 60_000)

  it('rejects a platform JWT whose hostRefs do not bind the ticket hostRef', async () => {
    const { runtimeApp } = createProxyApps(config())
    const foreign = sign(
      {
        sub: 'default/other-host',
        hostRefs: ['other-host'],
        workflowControlScopes: ['llm:grok:execute'],
        scope: 'workflow:approval:request',
      },
      'workflow-approvals'
    )
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${foreign}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
      })
    expect(res.status).toBe(403)
    expect(res.body.error).toBe('host_binding_mismatch')
  })

  it('rejects a refresh JWT that otherwise carries llm:grok:execute', async () => {
    const { runtimeApp } = createProxyApps(config())
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken({ scope: 'workflow:approval:refresh' })}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
      })
    expect(res.status).toBe(401)
    expect(res.body.error).toBe('Unauthorized')
  })

  it('rejects a platform JWT that omits the access-token scope', async () => {
    const { runtimeApp } = createProxyApps(config())
    const missingScope = sign(
      {
        sub: 'default/research-host',
        hostRefs: ['research-host'],
        workflowControlScopes: ['llm:grok:execute'],
      },
      'workflow-approvals'
    )
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${missingScope}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
      })
    expect(res.status).toBe(401)
    expect(res.body.error).toBe('Unauthorized')
  })

  it('rejects a platform JWT whose hostRefs is a wildcard', async () => {
    const { runtimeApp } = createProxyApps(config())
    const wildcard = sign(
      {
        sub: 'default/research-host',
        hostRefs: ['*'],
        workflowControlScopes: ['llm:grok:execute'],
        scope: 'workflow:approval:request',
      },
      'workflow-approvals'
    )
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${wildcard}`)
      .send({
        executionTicket: ticket(),
        requestHash: 'a'.repeat(64),
        request: {},
      })
    expect(res.status).toBe(401)
  })

  it('rejects execution tickets and admin permits without a numeric exp', () => {
    const cfg = config()
    const ticketNoExp = jwt.sign(
      {
        jti: '11111111-1111-4111-8111-111111111111',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash: 'a'.repeat(64),
        providerAttemptId: 'att-1',
      },
      privateKey,
      { algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
    )
    const permitNoExp = jwt.sign(
      { sub: 'admin-1', typ: 'grok-admin-permit', operation: 'catalog_list' },
      privateKey,
      { algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy-admin' }
    )
    expect(verifyExecutionTicket(ticketNoExp, cfg)).toBeNull()
    expect(verifyAdminPermit(permitNoExp, cfg)).toBeNull()
  })

  // R17-2: body admission can hold a body for up to maxQueueWaitMs before the
  // ticket is read, the same span as the ticket TTL, so an honest request can
  // reach this gate with an expired ticket. That answer must be the retryable
  // ticket_expired; every other ticket failure stays ticket_invalid.
  it('T-R17-2-grok answers ticket_expired for an authentic expired ticket and ticket_invalid otherwise', async () => {
    const { runtimeApp } = createProxyApps(config({ executionEnabled: false }))
    const { privateKey: foreignKey } = generateKeyPairSync('rsa', {
      modulusLength: 2048,
      publicKeyEncoding: { type: 'spki', format: 'pem' },
      privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
    })
    const expiredClaims = {
      jti: '11111111-1111-4111-8111-111111111111',
      typ: 'grok-execution-ticket',
      hostRef: 'research-host',
      model: 'gpt-5.1',
      requestHash: 'a'.repeat(64),
      providerAttemptId: 'att-1',
      exp: Math.floor(Date.now() / 1000) - 5,
    }
    const expiredTicket = (
      claims: Record<string, unknown>,
      key: string = privateKey,
      audience = 'grok-llm-proxy'
    ) => jwt.sign(claims, key, { algorithm: 'RS256', issuer: 'control-api', audience })
    const send = (executionTicket: string) =>
      request(runtimeApp)
        .post('/internal/runtime/v1/grok/completions')
        .set('Authorization', `Bearer ${platformToken()}`)
        .send({ executionTicket, requestHash: 'a'.repeat(64), request: {} })

    const expired = await send(expiredTicket(expiredClaims))
    expect(expired.status).toBe(403)
    expect(expired.body).toEqual({ error: 'ticket_expired' })

    // jsonwebtoken checks exp before audience and claims, so each of these is
    // expired too; none is an authentic Grok ticket.
    for (const forged of [
      expiredTicket(expiredClaims, foreignKey),
      expiredTicket(expiredClaims, privateKey, 'codex-llm-proxy'),
      expiredTicket({ ...expiredClaims, typ: 'codex-execution-ticket' }),
    ]) {
      const refused = await send(forged)
      expect(refused.status).toBe(403)
      expect(refused.body).toEqual({ error: 'ticket_invalid' })
    }

    // Witness: a live ticket passes the ticket gate and stops at the next one.
    const live = await send(ticket())
    expect(live.status).toBe(404)
    expect(live.body).toEqual({ error: 'disabled' })
  })

  it('rejects an admin permit whose operation does not match the route', async () => {
    const { adminApp } = createProxyApps(config())
    const wrong = sign(
      { sub: 'admin-1', typ: 'grok-admin-permit', operation: 'connection_test' },
      'grok-llm-proxy-admin'
    )
    const res = await request(adminApp)
      .post('/internal/admin/v1/grok/models')
      .set('Authorization', `Bearer ${wrong}`)
      .send({ accessToken: 'tok' })
    expect(res.status).toBe(401)
  })

  it('rejects missing, zero, and unbounded config', () => {
    expect(() => loadConfig({ GROK_LLM_PROXY_RUNTIME_PORT: '0' })).toThrow(/greater than zero/)
    expect(() =>
      loadConfig({
        GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
        GROK_LLM_PROXY_MAX_BODY_BYTES: '0',
      })
    ).toThrow(/greater than zero/)
    expect(() => loadConfig({})).toThrow(/PEM-encoded public key/)
    expect(() =>
      loadConfig({
        GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
        GROK_LLM_PROXY_MAX_STREAM_DURATION_MS: String(Number.MAX_SAFE_INTEGER),
      })
    ).toThrow(/bounded positive integer/)
    expect(() =>
      loadConfig({
        GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
        GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '0',
      })
    ).toThrow(/GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS must be a finite integer greater than zero/)
  })

  it('defaults the upstream idle timeout to the published STREAM_LIMITS value', () => {
    const loaded = loadConfig({
      GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
      GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
      GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
    })
    expect(loaded.upstreamIdleTimeoutMs).toBe(600_000)
    expect(
      loadConfig({
        GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
        GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
        GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
        GROK_LLM_PROXY_UPSTREAM_IDLE_TIMEOUT_MS: '45000',
      }).upstreamIdleTimeoutMs
    ).toBe(45_000)
  })

  it('defaults the total stream cap and the deadline ceiling to 30 min', () => {
    const loaded = loadConfig({
      GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
      GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
      GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
    })
    expect(loaded.maxStreamDurationMs).toBe(1_800_000)
    expect(loaded.maxDeadlineMs).toBe(1_800_000)
  })

  it('defaults the SSE heartbeat to 15 s and rejects a non-positive interval', () => {
    const env = {
      GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
      GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
      GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
    }
    expect(loadConfig(env).heartbeatIntervalMs).toBe(15_000)
    expect(
      loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '5000' }).heartbeatIntervalMs
    ).toBe(5_000)
    for (const raw of ['0', '-1']) {
      expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
        /GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be a finite integer greater than zero/
      )
    }
  })

  it('refuses at startup a heartbeat interval the Host HTTP client would time out on', () => {
    const env = {
      GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
      GROK_LLM_PROXY_CONTROL_API_URL: 'http://control-api:8080',
      GROK_LLM_PROXY_CONTROL_API_TOKEN: 'service-token',
    }
    // The bound itself is accepted, so the refusal below is the bound and not
    // a parse failure.
    expect(
      loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: '60000' }).heartbeatIntervalMs
    ).toBe(60_000)
    for (const raw of ['60001', '600000']) {
      expect(() => loadConfig({ ...env, GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS: raw })).toThrow(
        /GROK_LLM_PROXY_HEARTBEAT_INTERVAL_MS must be at most 60000/
      )
    }
  })
})

describe('grok-llm-proxy execution kill switch', () => {
  const lookup = async () => [{ address: '1.2.3.4', family: 4 }]

  function validCompletionBody(): Record<string, unknown> {
    const raw = {
      schemaVersion: 'grok-completion-request.v1',
      requestId: 'req-kill-switch',
      idempotencyKey: 'idem-kill-switch',
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'hello' }],
    }
    const parsed = parseGrokCompletionRequestV1(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    const executionTicket = sign(
      {
        jti: '33333333-3333-4333-8333-333333333333',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId: 'att-kill-switch',
      },
      'grok-llm-proxy'
    )
    return { executionTicket, requestHash, request: raw }
  }

  function recordingClient(): { client: ControlApiClient; redeems: unknown[] } {
    const redeems: unknown[] = []
    const client = {
      async redeem(input: unknown) {
        redeems.push(input)
        throw new ControlApiClientError('no_grant', 'no grant in kill-switch test')
      },
      async finalize() {
        throw new Error('finalize must not run in kill-switch test')
      },
    } as unknown as ControlApiClient
    return { client, redeems }
  }

  it('returns 404 disabled for a fully valid runtime JWT + ticket when execution is disabled', async () => {
    const body = validCompletionBody()

    // Liveness witness: the identical request reaches redeem when enabled.
    const enabled = recordingClient()
    const live = createProxyApps(config({ executionEnabled: true, maxBodyBytes: 65_536 }), {
      controlApiClient: enabled.client,
      lookup,
    })
    const reached = await request(live.runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send(body)
    expect(reached.status).toBe(403)
    expect(JSON.parse(reached.text)).toEqual({ error: 'no_grant' })
    expect(enabled.redeems).toHaveLength(1)

    const disabled = recordingClient()
    const off = createProxyApps(config({ executionEnabled: false, maxBodyBytes: 65_536 }), {
      controlApiClient: disabled.client,
      lookup,
    })
    const res = await request(off.runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send(body)
    expect(res.status).toBe(404)
    expect(res.body).toEqual({ error: 'disabled' })
    expect(disabled.redeems).toHaveLength(0)
  })

  it('returns 404 disabled for a valid admin permit when execution is disabled', async () => {
    const upstream = () =>
      new Response(JSON.stringify({ data: [{ id: 'gpt-5.1' }] }), {
        status: 200,
        headers: { 'content-type': 'application/json' },
      })

    // Liveness witness: the identical admin call reaches the catalog upstream when enabled.
    let enabledCalls = 0
    const live = createProxyApps(config({ executionEnabled: true, maxBodyBytes: 65_536 }), {
      fetchFn: (async () => {
        enabledCalls += 1
        return upstream()
      }) as typeof fetch,
      lookup,
    })
    const reached = await request(live.adminApp)
      .post('/internal/admin/v1/grok/models')
      .set('Authorization', `Bearer ${adminPermit()}`)
      .send({ accessToken: 'tok' })
    expect(reached.status).toBe(200)
    expect(reached.body.outcome).toBe('ready')
    expect(enabledCalls).toBe(1)

    let disabledCalls = 0
    const off = createProxyApps(config({ executionEnabled: false, maxBodyBytes: 65_536 }), {
      fetchFn: (async () => {
        disabledCalls += 1
        return upstream()
      }) as typeof fetch,
      lookup,
    })
    const res = await request(off.adminApp)
      .post('/internal/admin/v1/grok/models')
      .set('Authorization', `Bearer ${adminPermit()}`)
      .send({ accessToken: 'tok' })
    expect(res.status).toBe(404)
    expect(res.body).toEqual({ error: 'disabled' })
    expect(disabledCalls).toBe(0)
  })
})

describe('grok-llm-proxy startup config', () => {
  const base = {
    GROK_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
    GROK_LLM_PROXY_CONTROL_API_URL:
      'http://control-api-rpc-gateway.control-plane.svc.cluster.local:8090/api/v1',
    GROK_LLM_PROXY_CONTROL_API_TOKEN: 'dev-grok-llm-proxy-token',
  }

  it('loads a complete environment', () => {
    const loaded = loadConfig(base)
    expect(loaded.controlApiBaseUrl).toBe(base.GROK_LLM_PROXY_CONTROL_API_URL)
    expect(loaded.controlApiServiceToken).toBe('dev-grok-llm-proxy-token')
    expect(loaded.controlApiServiceName).toBe('grok-llm-proxy')
  })

  // #731 — the envelope carries the contract-capped `request` plus the ticket,
  // the hash and the deadline. A body limit equal to the contract cap refuses,
  // as a 413, requests the contract itself accepts.
  it('T-R2-6b-grok defaults the body limit to the contract request cap plus a 16 KiB envelope allowance', () => {
    expect(loadConfig(base).maxBodyBytes).toBe(LIMITS.maxRequestBodyBytes + 16 * 1024)
  })

  // R11 — the allowance has one source, the contract, shared with control-api
  // and mcp-host. A literal here could drift from theirs.
  it('T-R11-grok takes the envelope allowance from the contract', () => {
    expect(CONTRACT_ENVELOPE_ALLOWANCE_BYTES).toBe(16 * 1024)
    expect(ENVELOPE_ALLOWANCE_BYTES).toBe(CONTRACT_ENVELOPE_ALLOWANCE_BYTES)
  })

  // R9-3 (L-3) — a lower override would answer 413 to requests the contract
  // accepts, so it is refused at startup.
  it('T-R9-3-grok refuses a body limit below the contract request cap plus the envelope allowance', () => {
    const floor = LIMITS.maxRequestBodyBytes + ENVELOPE_ALLOWANCE_BYTES
    expect(() => loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor - 1) })).toThrow(
      'GROK_LLM_PROXY_MAX_BODY_BYTES must be at least the contract request cap plus the envelope allowance'
    )
    expect(() =>
      loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(LIMITS.maxRequestBodyBytes) })
    ).toThrow(/GROK_LLM_PROXY_MAX_BODY_BYTES must be at least/)
    // Witness: the floor itself and any larger value load.
    expect(loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor) }).maxBodyBytes).toBe(
      floor
    )
    expect(
      loadConfig({ ...base, GROK_LLM_PROXY_MAX_BODY_BYTES: String(floor + 1) }).maxBodyBytes
    ).toBe(floor + 1)
  })

  it('defaults the visual body limit to the contract visual ceiling, with no allowance on top', () => {
    expect(LIMITS.maxVisualRequestBodyBytes).toBe(36_700_160)
    expect(loadConfig(base).maxVisualBodyBytes).toBe(LIMITS.maxVisualRequestBodyBytes)
  })

  it('refuses a visual body limit below the contract visual ceiling', () => {
    expect(() =>
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes - 1),
      })
    ).toThrow(
      `GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at least ${LIMITS.maxVisualRequestBodyBytes}, the contract maxVisualRequestBodyBytes`
    )
    expect(() => loadConfig({ ...base, GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: '1024' })).toThrow(
      /^GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at least 36700160,/
    )
    // Witness: the ceiling itself loads.
    expect(
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes),
      }).maxVisualBodyBytes
    ).toBe(LIMITS.maxVisualRequestBodyBytes)
  })

  // The 2048Mi memory limit was measured with one visual stream at the contract
  // ceiling. A larger visual limit would admit bodies that measurement never
  // covered, so the proxy refuses to start with one.
  it('refuses a visual body limit above the contract visual ceiling', () => {
    expect(() =>
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes + 1),
      })
    ).toThrow(
      `GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at most ${LIMITS.maxVisualRequestBodyBytes}, the contract maxVisualRequestBodyBytes`
    )
    expect(() =>
      loadConfig({ ...base, GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(64 * 1024 * 1024) })
    ).toThrow(/^GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be at most 36700160,/)
    // Witness: the ceiling itself loads.
    expect(
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(LIMITS.maxVisualRequestBodyBytes),
      }).maxVisualBodyBytes
    ).toBe(LIMITS.maxVisualRequestBodyBytes)
  })

  // With an equal budget every visual envelope would also fit the ordinary
  // parser's cap, so the visual gate's separate admission and memory
  // accounting would never engage.
  it('refuses a visual body limit less than or equal to the ordinary body limit', () => {
    const equal = LIMITS.maxVisualRequestBodyBytes
    expect(() =>
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal),
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
      })
    ).toThrow(
      'GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be greater than GROK_LLM_PROXY_MAX_BODY_BYTES'
    )
    expect(() =>
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal + 1),
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
      })
    ).toThrow(
      'GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES must be greater than GROK_LLM_PROXY_MAX_BODY_BYTES'
    )
    // Liveness witness: a visual limit one byte above the ordinary limit loads.
    // The visual limit is pinned to the contract ceiling, so the ordinary limit
    // moves instead.
    expect(
      loadConfig({
        ...base,
        GROK_LLM_PROXY_MAX_BODY_BYTES: String(equal - 1),
        GROK_LLM_PROXY_MAX_VISUAL_BODY_BYTES: String(equal),
      }).maxVisualBodyBytes
    ).toBe(equal)
  })

  it('T-R2-6c-grok does not refuse a request at the contract cap with a real ticket as payload_too_large', async () => {
    const { runtimeApp } = createProxyApps(config({ maxBodyBytes: loadConfig(base).maxBodyBytes }))
    const atCap = { pad: 'x'.repeat(LIMITS.maxRequestBodyBytes - '{"pad":""}'.length) }
    expect(Buffer.byteLength(JSON.stringify(atCap), 'utf8')).toBe(LIMITS.maxRequestBodyBytes)
    const res = await request(runtimeApp)
      .post('/internal/runtime/v1/grok/completions')
      .set('Authorization', `Bearer ${platformToken()}`)
      .send({ executionTicket: ticket(), requestHash: 'a'.repeat(64), request: atCap })
    expect(res.body.error).not.toBe('payload_too_large')
    expect(res.status).not.toBe(413)
    // Witness: the body parser accepted the envelope and the route answered.
    expect(typeof res.body.error).toBe('string')
  })

  it.each([undefined, '', '   '])('fails at startup when the control-api URL is %j', value => {
    expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
      /GROK_LLM_PROXY_CONTROL_API_URL/
    )
  })

  it.each(['not a url', 'ftp://control-api/api/v1', 'file:///etc/passwd'])(
    'fails at startup when the control-api URL %j is not http(s)',
    value => {
      expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_URL: value })).toThrow(
        /GROK_LLM_PROXY_CONTROL_API_URL/
      )
    }
  )

  it.each([undefined, '', '   '])(
    'fails at startup when the control-api service token is %j',
    value => {
      expect(() => loadConfig({ ...base, GROK_LLM_PROXY_CONTROL_API_TOKEN: value })).toThrow(
        /GROK_LLM_PROXY_CONTROL_API_TOKEN/
      )
    }
  )
})

describe('grok-llm-proxy attempt telemetry', () => {
  const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
  // Keys that carry caller content. The attempt line is identifiers and counts;
  // none of these may ever reach it. The four names this suite has always
  // checked are joined by every leaf the service itself redacts, so a value the
  // logger would scrub cannot be declared safe here merely because this list
  // forgot it — `accessToken` and `attemptReceipt` both come back from the
  // redeem fixture below.
  const FORBIDDEN_LOG_KEYS = [
    ...new Set([
      'body',
      'request',
      'executionTicket',
      'arguments',
      ...REDACT_PATHS.map(
        redacted =>
          redacted
            .replace(/\['([^']+)'\]/g, '.$1')
            .split('.')
            .pop() as string
      ).filter(key => key !== '*'),
    ]),
  ]

  function completionBody(
    providerAttemptId: string,
    tamper?: (raw: Record<string, unknown>) => void
  ): Record<string, unknown> {
    const raw: Record<string, unknown> = {
      schemaVersion: 'grok-completion-request.v1',
      requestId: `req-${providerAttemptId}`,
      idempotencyKey: `idem-${providerAttemptId}`,
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'hello' }],
    }
    const parsed = parseGrokCompletionRequestV1(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    // Applied after hashing: the ticket stays bound to the untampered request,
    // so the transport's parser rejects the body before any hash comparison.
    tamper?.(raw)
    const executionTicket = sign(
      {
        jti: '55555555-5555-4555-8555-555555555555',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId,
      },
      'grok-llm-proxy'
    )
    return { executionTicket, requestHash, request: raw }
  }

  function grantingClient(maxStreamDurationMs = 1_800_000): {
    client: ControlApiClient
    receipts: unknown[]
  } {
    const receipts: unknown[] = []
    const client = {
      async redeem(): Promise<RedeemAttemptSuccess> {
        return {
          accessToken: 'test-access-telemetry',
          transport: {
            protocolVersion: 'grok-subscription-transport.v1',
            completionsOrigin: GROK_COMPLETIONS_ORIGIN,
            catalogOrigin: GROK_CATALOG_ORIGIN,
            operation: 'completion_stream',
            servedModel: 'gpt-5.1',
            maxStreamDurationMs,
          },
          expiryClass: 'short_lived',
          attemptReceipt: 'c'.repeat(64),
        }
      },
      async finalize(input: {
        receipt: { outcome: FinalizeAttemptSuccess['outcome'] }
      }): Promise<FinalizeAttemptSuccess> {
        receipts.push(input.receipt)
        return { providerAttemptId: 'att', outcome: input.receipt.outcome, duplicate: false }
      },
    } as unknown as ControlApiClient
    return { client, receipts }
  }

  function upstream(textDeltas: number, calls: number): typeof fetch {
    const frames: string[] = []
    for (let index = 0; index < textDeltas; index += 1) {
      frames.push(
        `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${index}` })}\n\n`
      )
    }
    for (let index = 0; index < calls; index += 1) {
      frames.push(
        `data: ${JSON.stringify({
          type: 'response.output_item.done',
          item: { type: 'function_call', id: `call-${index}`, name: 'lookup', arguments: '{}' },
        })}\n\n`
      )
    }
    frames.push('data: {"type":"response.completed"}\n\n')
    return (async () =>
      new Response(frames.join(''), {
        status: 200,
        headers: { 'content-type': 'text/event-stream' },
      })) as typeof fetch
  }

  // One SSE block larger than the transport's 1 MiB buffer and never terminated,
  // so the buffer guard fires before any frame is delivered.
  function unterminatedUpstream(): typeof fetch {
    return (async () =>
      new Response(`data: ${'x'.repeat(1_100_000)}`, {
        status: 200,
        headers: { 'content-type': 'text/event-stream' },
      })) as typeof fetch
  }

  // One tool call whose `arguments` deltas add up past the retained budget,
  // delivered one frame per read so the 1 MiB SSE buffer guard — which only
  // bounds the unparsed tail between two `\n\n` boundaries — never fires and
  // the argument budget is the guard under test.
  const ARGUMENT_OVERRUN_CHUNK = 65_536

  function oversizedArgumentsUpstream(): typeof fetch {
    const chunks = MAX_TOOL_CALL_ARGUMENT_BYTES / ARGUMENT_OVERRUN_CHUNK + 1
    const frames = [
      `data: ${JSON.stringify({
        type: 'response.output_item.added',
        item: { type: 'function_call', id: 'call-args', name: 'lookup', arguments: '' },
      })}\n\n`,
      ...Array.from(
        { length: chunks },
        () =>
          `data: ${JSON.stringify({
            type: 'response.function_call_arguments.delta',
            item_id: 'call-args',
            delta: 'a'.repeat(ARGUMENT_OVERRUN_CHUNK),
          })}\n\n`
      ),
      'data: {"type":"response.completed"}\n\n',
    ]
    const encoder = new TextEncoder()
    return (async () => {
      let index = 0
      const body = new ReadableStream<Uint8Array>({
        pull(controller) {
          const next = frames[index]
          index += 1
          if (next === undefined) controller.close()
          else controller.enqueue(encoder.encode(next))
        },
      })
      return new Response(body, {
        status: 200,
        headers: { 'content-type': 'text/event-stream' },
      })
    }) as typeof fetch
  }

  // One call whose `arguments` are truncated JSON, closed by a completed response.
  function malformedArgumentsUpstream(): typeof fetch {
    const frames = [
      `data: ${JSON.stringify({
        type: 'response.output_item.done',
        item: { type: 'function_call', id: 'call-bad', name: 'lookup', arguments: '{"q":' },
      })}\n\n`,
      'data: {"type":"response.completed"}\n\n',
    ]
    return (async () =>
      new Response(frames.join(''), {
        status: 200,
        headers: { 'content-type': 'text/event-stream' },
      })) as typeof fetch
  }

  // The upstream's context-window refusal recorded on 2026-09-23 (R10): an
  // HTTP 400 before any stream, with the marker inside the `error` string.
  function contextOverflowUpstream(): typeof fetch {
    return (async () =>
      new Response(
        JSON.stringify({
          code: 'invalid-argument',
          error:
            "Failed to start sampling: [input_too_large] The prompt is too long for this model's context window (1107771 tokens > 500000 tokens)",
        }),
        { status: 400, headers: { 'content-type': 'application/json' } }
      )) as typeof fetch
  }

  function denyingClient(code: string): ControlApiClient {
    return {
      async redeem() {
        throw new ControlApiClientError(code, 'control API request denied')
      },
      async finalize() {
        throw new Error('finalize must not run after a denied redeem')
      },
    } as unknown as ControlApiClient
  }

  // Sends `textDeltas` frames, then never sends another byte. The body ignores
  // the fetch signal, like a peer that stops writing without closing.
  function stalledUpstream(textDeltas: number): typeof fetch {
    return (async () => {
      const encoder = new TextEncoder()
      let sent = 0
      const body = new ReadableStream<Uint8Array>(
        {
          pull(controller) {
            if (sent < textDeltas) {
              controller.enqueue(
                encoder.encode(
                  `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${sent}` })}\n\n`
                )
              )
              sent += 1
              return undefined
            }
            return new Promise<void>(() => undefined)
          },
        },
        { highWaterMark: 0 }
      )
      return new Response(body, {
        status: 200,
        headers: { 'content-type': 'text/event-stream' },
      })
    }) as typeof fetch
  }

  function failureCount(metricsText: string, code: string): number {
    const line = metricsText
      .split('\n')
      .find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))
    return line ? Number(line.split(' ').pop()) : 0
  }

  function timeoutCount(metricsText: string, kind: 'idle' | 'total'): number {
    const line = metricsText
      .split('\n')
      .find(row => row.startsWith(`grok_proxy_upstream_timeouts_total{kind="${kind}"}`))
    return line ? Number(line.split(' ').pop()) : 0
  }

  function keepaliveCount(text: string): number {
    return text.split(': keepalive\n\n').length - 1
  }

  // Stays silent for `silentMs` after the fetch resolves, then streams one text
  // delta and completes, like a model that reasons before its first token.
  function silentThenCompletingUpstream(silentMs: number): typeof fetch {
    return (async () =>
      new Response(
        new ReadableStream<Uint8Array>({
          async start(controller) {
            await new Promise(resolve => setTimeout(resolve, silentMs))
            controller.enqueue(
              new TextEncoder().encode(
                'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
                  'data: {"type":"response.completed"}\n\n'
              )
            )
            controller.close()
          },
        }),
        { status: 200, headers: { 'content-type': 'text/event-stream' } }
      )) as typeof fetch
  }

  // Answers `status` with no SSE body after `delayMs`.
  function slowFailingUpstream(delayMs: number, status: number): typeof fetch {
    return (async () => {
      await new Promise(resolve => setTimeout(resolve, delayMs))
      return new Response('upstream failed', { status })
    }) as typeof fetch
  }

  // Denies the redeem after `delayMs`, which is longer than the heartbeat
  // interval the caller configures.
  function slowDenyingClient(
    code: string,
    delayMs: number
  ): {
    client: ControlApiClient
    redeemCalls: () => number
  } {
    let calls = 0
    const client = {
      async redeem() {
        calls += 1
        await new Promise(resolve => setTimeout(resolve, delayMs))
        throw new ControlApiClientError(code, 'control API request denied')
      },
      async finalize() {
        throw new Error('finalize must not run after a denied redeem')
      },
    } as unknown as ControlApiClient
    return { client, redeemCalls: () => calls }
  }

  async function run(options: {
    providerAttemptId: string
    textDeltas?: number
    calls?: number
    tamper?: (raw: Record<string, unknown>) => void
    deniedCode?: string
    fetchFn?: typeof fetch
    maxStreamDurationMs?: number
    configOverrides?: Partial<GrokLlmProxyConfig>
    controlApiClient?: ControlApiClient
    lookup?: (hostname: string) => Promise<Array<{ address: string; family: number }>>
  }) {
    const info = vi.spyOn(logger, 'info')
    const warn = vi.spyOn(logger, 'warn')
    const { client, receipts } = grantingClient(options.maxStreamDurationMs)
    const apps = createProxyApps(config({ maxBodyBytes: 65_536, ...options.configOverrides }), {
      controlApiClient:
        options.controlApiClient ??
        (options.deniedCode ? denyingClient(options.deniedCode) : client),
      fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),
      lookup: options.lookup ?? lookup,
    })
    try {
      const res = await request(apps.runtimeApp)
        .post('/internal/runtime/v1/grok/completions')
        .set('Authorization', `Bearer ${platformToken()}`)
        .send(completionBody(options.providerAttemptId, options.tamper))
      const metricsText = (await request(apps.probeApp).get('/metrics')).text
      const logged = [...info.mock.calls, ...warn.mock.calls].map(
        call => call[0] as unknown as Record<string, unknown>
      )
      const lines = logged.filter(entry => entry?.event === 'grok_proxy_attempt_finished')
      const unreachable = logged.filter(
        entry => entry?.event === 'grok_proxy_control_api_unreachable'
      )
      const causeCodeLines = logged.filter(entry => entry && 'causeCode' in entry)
      return { res, receipts, lines, unreachable, causeCodeLines, metricsText }
    } finally {
      info.mockRestore()
      warn.mockRestore()
    }
  }

  /** Returns how many keys it inspected, so an empty payload cannot pass. */
  function walkForbiddenKeys(value: unknown, trail: string): number {
    if (Array.isArray(value)) {
      return value.reduce<number>(
        (total, item, index) => total + walkForbiddenKeys(item, `${trail}[${index}]`),
        0
      )
    }
    if (value === null || typeof value !== 'object') return 0
    let visited = 0
    for (const [key, nested] of Object.entries(value)) {
      visited += 1
      if (FORBIDDEN_LOG_KEYS.includes(key)) {
        throw new Error(`log line carries the forbidden key ${trail}.${key}`)
      }
      visited += walkForbiddenKeys(nested, `${trail}.${key}`)
    }
    return visited
  }

  function expectNoForbiddenKeys(line: Record<string, unknown>): void {
    // A flat check would clear `details` and `usage` without ever opening them.
    const visited = walkForbiddenKeys(line, 'line')
    // Liveness witness: a walk over an empty or non-object payload inspects
    // nothing and would otherwise report the line as clean.
    expect(visited).toBeGreaterThan(0)
  }

  // G1-1 (#720): an upstream 429 reaches the Host as a 429 rate_limited with
  // the upstream's Retry-After, not as a retryable 503 provider_unavailable.
  function rateLimitedUpstream(retryAfter?: string): typeof fetch {
    return (async () =>
      new Response('slow down', {
        status: 429,
        headers: retryAfter === undefined ? {} : { 'retry-after': retryAfter },
      })) as typeof fetch
  }

  it('logs the error of an unmapped handler failure while it still answers 503 provider_unavailable', async () => {
    const boom = new Error('boom')
    const failingClient = {
      async redeem(): Promise<RedeemAttemptSuccess> {
        throw boom
      },
      async finalize(): Promise<FinalizeAttemptSuccess> {
        throw new Error('finalize must not run after a failed redeem')
      },
    } as unknown as ControlApiClient
    const { res, lines } = await run({
      providerAttemptId: 'att-unmapped-error',
      controlApiClient: failingClient,
    })
    // Witnesses: the request reached the handler's catch and was logged once.
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'provider_unavailable' })
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-unmapped-error',
      outcome: 'failed',
      code: 'provider_unavailable',
      httpStatus: 503,
    })
    expect(lines[0]?.err).toBe(boom)
  })

  it('logs a mapped transport failure without an err entry', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-mapped-transport-error',
      fetchFn: rateLimitedUpstream('7'),
    })
    // Witnesses: the failure reached the same catch and produced its attempt line.
    expect(res.status).toBe(429)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-mapped-transport-error',
      outcome: 'failed',
      code: 'rate_limited',
    })
    expect('err' in (lines[0] ?? {})).toBe(false)
  })

  async function closedLoopbackPort(): Promise<number> {
    const closed = createServer()
    await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
    const { port } = closed.address() as AddressInfo
    await new Promise<void>(resolve => closed.close(() => resolve()))
    return port
  }

  // Real undici fetch against a loopback port nothing listens on, so the
  // failure is undici's own TypeError, whose cause message names the address.
  function closedPortFetch(port: number, calls: { count: number }): typeof fetch {
    return (async (_input: unknown, init?: RequestInit) => {
      calls.count += 1
      return fetch(`http://127.0.0.1:${port}/`, {
        method: init?.method ?? 'GET',
        signal: init?.signal ?? null,
      })
    }) as typeof fetch
  }

  // Serializes Error values with their message and cause, as the logger's err
  // serializer does, so an error hidden in a log line cannot pass as clean.
  function serializeWithErrors(value: unknown): string {
    return JSON.stringify(value, (_key, nested: unknown) =>
      nested instanceof Error
        ? { name: nested.name, message: nested.message, cause: nested.cause }
        : nested
    )
  }

  // Review R3-L1: a completion fetch nothing accepted reaches the handler as a
  // mapped transport error, so the attempt line keeps the cause code and drops
  // the err entry whose cause message names the upstream address.
  it('(r3-l1a) logs an upstream connection refusal by cause code, without err or address', async () => {
    const port = await closedLoopbackPort()
    const calls = { count: 0 }
    const { res, lines } = await run({
      providerAttemptId: 'att-upstream-refused',
      fetchFn: closedPortFetch(port, calls),
    })
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'provider_unavailable' })
    // Witnesses: the real fetch ran against the closed port, and the failure
    // produced its attempt line.
    expect(calls.count).toBe(1)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-upstream-refused',
      outcome: 'failed',
      code: 'provider_unavailable',
      reason: 'upstream fetch failed',
      causeCode: 'ECONNREFUSED',
      deliveredAs: 'http_status',
      httpStatus: 503,
    })
    expect('err' in lines[0]!).toBe(false)
    expect(serializeWithErrors(lines[0])).not.toContain(`127.0.0.1:${port}`)
    expectNoForbiddenKeys(lines[0]!)
  })

  // Review R3-L9: the admin route's refusal line carries only the code, so the
  // catalog logs the cause code of a failed fetch before the 503.
  it('(r3-l9a) logs a catalog connection refusal by cause code on both admin routes', async () => {
    const port = await closedLoopbackPort()
    for (const [route, operation] of [
      ['/internal/admin/v1/grok/models', 'catalog_list'],
      ['/internal/admin/v1/grok/test', 'connection_test'],
    ] as const) {
      const calls = { count: 0 }
      const warn = vi.spyOn(logger, 'warn')
      const info = vi.spyOn(logger, 'info')
      try {
        const { adminApp } = createProxyApps(
          config({ executionEnabled: true, maxBodyBytes: 65_536 }),
          { fetchFn: closedPortFetch(port, calls), lookup }
        )
        const res = await request(adminApp)
          .post(route)
          .set(
            'Authorization',
            `Bearer ${sign({ sub: 'admin-1', typ: 'grok-admin-permit', operation }, 'grok-llm-proxy-admin')}`
          )
          .send({ accessToken: 'tok' })
        expect(res.status).toBe(503)
        expect(res.body).toEqual({ error: 'provider_unavailable' })
        // Witness: the real fetch ran against the closed port.
        expect(calls.count).toBe(1)
        const logged = [...warn.mock.calls, ...info.mock.calls].map(
          call => call[0] as unknown as Record<string, unknown>
        )
        expect(logged.filter(entry => entry?.event === 'grok_catalog_upstream')).toEqual([
          { event: 'grok_catalog_upstream', causeCode: 'ECONNREFUSED' },
        ])
        expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
          { event: 'grok_proxy_denied', code: 'provider_unavailable' },
        ])
        expect(serializeWithErrors(logged)).not.toContain(`127.0.0.1:${port}`)
      } finally {
        warn.mockRestore()
        info.mockRestore()
      }
    }
  })

  it('(r3-l9b) answers a catalog body that is not JSON with a mapped 503 and its own log line', async () => {
    let calls = 0
    const warn = vi.spyOn(logger, 'warn')
    try {
      const { adminApp } = createProxyApps(
        config({ executionEnabled: true, maxBodyBytes: 65_536 }),
        {
          fetchFn: (async () => {
            calls += 1
            return new Response('<html>maintenance</html>', {
              status: 200,
              headers: { 'content-type': 'text/html' },
            })
          }) as typeof fetch,
          lookup,
        }
      )
      const res = await request(adminApp)
        .post('/internal/admin/v1/grok/models')
        .set('Authorization', `Bearer ${adminPermit()}`)
        .send({ accessToken: 'tok' })
      expect(res.status).toBe(503)
      expect(res.body).toEqual({ error: 'provider_unavailable' })
      expect(calls).toBe(1)
      const logged = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
      expect(logged.filter(entry => entry?.event === 'grok_catalog_upstream')).toEqual([
        { event: 'grok_catalog_upstream', reason: 'invalid_json' },
      ])
      expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
        { event: 'grok_proxy_denied', code: 'provider_unavailable' },
      ])
    } finally {
      warn.mockRestore()
    }
  })

  // Review R4-L1: a DNS failure carries its code on the lookup error itself,
  // not in `cause`, and its message names the upstream host. The attempt line
  // and both admin routes log it by cause code, with no err entry and no host.
  // These real .invalid lookups depend on the resolver honoring NXDOMAIN.
  const INVALID_UPSTREAM_HOST = 'grok-r4-l1.invalid'

  function invalidHostLookup(calls: { count: number }) {
    return async () => {
      calls.count += 1
      const { lookup: dnsLookup } = await import('node:dns/promises')
      const records = await dnsLookup(INVALID_UPSTREAM_HOST, { all: true })
      return records.map(record => ({ address: record.address, family: record.family }))
    }
  }

  function countingFetch(calls: { count: number }): typeof fetch {
    return (async () => {
      calls.count += 1
      throw new Error('fetch must not run after a failed lookup')
    }) as typeof fetch
  }

  it('(r4-l1a) logs a DNS lookup failure by cause code, without err or host', async () => {
    const lookups = { count: 0 }
    const fetches = { count: 0 }
    const { res, lines } = await run({
      providerAttemptId: 'att-upstream-dns',
      fetchFn: countingFetch(fetches),
      lookup: invalidHostLookup(lookups),
    })
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'provider_unavailable' })
    // Witnesses: the real lookup ran once, failed before any fetch, and the
    // failure produced its attempt line.
    expect(lookups.count).toBe(1)
    expect(fetches.count).toBe(0)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-upstream-dns',
      outcome: 'failed',
      code: 'provider_unavailable',
      reason: 'upstream fetch failed',
      deliveredAs: 'http_status',
      httpStatus: 503,
    })
    expect(String(lines[0]!.causeCode)).toMatch(/^(ENOTFOUND|EAI_AGAIN)$/)
    expect('err' in lines[0]!).toBe(false)
    expect(serializeWithErrors(lines[0])).not.toContain(INVALID_UPSTREAM_HOST)
    expectNoForbiddenKeys(lines[0]!)
  }, 15_000)

  // Both admin routes look up serially: 30 s bounds two 15 s lookup budgets.
  it('(r4-l1b) logs a catalog DNS lookup failure by cause code on both admin routes', async () => {
    for (const [route, operation] of [
      ['/internal/admin/v1/grok/models', 'catalog_list'],
      ['/internal/admin/v1/grok/test', 'connection_test'],
    ] as const) {
      const lookups = { count: 0 }
      const fetches = { count: 0 }
      const warn = vi.spyOn(logger, 'warn')
      const info = vi.spyOn(logger, 'info')
      const error = vi.spyOn(logger, 'error')
      try {
        const { adminApp } = createProxyApps(
          config({ executionEnabled: true, maxBodyBytes: 65_536 }),
          { fetchFn: countingFetch(fetches), lookup: invalidHostLookup(lookups) }
        )
        const res = await request(adminApp)
          .post(route)
          .set(
            'Authorization',
            `Bearer ${sign({ sub: 'admin-1', typ: 'grok-admin-permit', operation }, 'grok-llm-proxy-admin')}`
          )
          .send({ accessToken: 'tok' })
        expect(res.status).toBe(503)
        expect(res.body).toEqual({ error: 'provider_unavailable' })
        // Witnesses: the real lookup ran once and failed before any fetch.
        expect(lookups.count).toBe(1)
        expect(fetches.count).toBe(0)
        const logged = [...warn.mock.calls, ...info.mock.calls, ...error.mock.calls].map(
          call => call[0] as unknown as Record<string, unknown>
        )
        const upstream = logged.filter(entry => entry?.event === 'grok_catalog_upstream')
        expect(upstream).toHaveLength(1)
        expect(Object.keys(upstream[0]!).sort()).toEqual(['causeCode', 'event'])
        expect(String(upstream[0]!.causeCode)).toMatch(/^(ENOTFOUND|EAI_AGAIN)$/)
        expect(logged.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
          { event: 'grok_proxy_denied', code: 'provider_unavailable' },
        ])
        expect(serializeWithErrors(logged)).not.toContain(INVALID_UPSTREAM_HOST)
      } finally {
        warn.mockRestore()
        info.mockRestore()
        error.mockRestore()
      }
    }
  }, 30_000)

  it('logs a mapped control-api failure without an err entry', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-mapped-control-api-error',
      deniedCode: 'ticket_expired',
    })
    expect(res.status).toBeGreaterThanOrEqual(400)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-mapped-control-api-error',
      outcome: 'failed',
      code: 'ticket_expired',
    })
    expect('err' in (lines[0] ?? {})).toBe(false)
  })

  it.each(['visual_host_share', 'visual_gate', 'proxy_capacity_exceeded'])(
    'normalizes real transport upstream 503 claiming %s to provider_unavailable',
    async code => {
      const fetchFn = vi
        .fn<typeof fetch>()
        .mockResolvedValue(
          Response.json({ error: code, reason: 'local capacity claim' }, { status: 503 })
        )
      const { res, receipts, lines, metricsText } = await run({
        providerAttemptId: 'att-spoofed-local',
        fetchFn,
      })
      expect(fetchFn).toHaveBeenCalledTimes(1)
      expect(res.status).toBe(503)
      expect(res.body).toEqual({ error: 'provider_unavailable' })
      expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
      expect(lines).toHaveLength(1)
      expect(lines[0]).toMatchObject({
        code: 'provider_unavailable',
        deliveredAs: 'http_status',
        httpStatus: 503,
      })
      expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
      expect(failureCount(metricsText, 'request_limit')).toBe(0)
    }
  )

  it('(g1-1a) answers 429 rate_limited and forwards a valid upstream Retry-After', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-rate-http',
      fetchFn: rateLimitedUpstream('7'),
    })
    expect(res.status).toBe(429)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.headers['retry-after']).toBe('7')
    expect(res.body).toEqual({ error: 'rate_limited' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-rate-http',
      outcome: 'failed',
      code: 'rate_limited',
      details: { retryAfterSeconds: 7 },
      deliveredAs: 'http_status',
      httpStatus: 429,
    })
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'rate_limited')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  it('(g1-1b) answers 429 with no Retry-After header when the upstream value is invalid', async () => {
    const { res } = await run({
      providerAttemptId: 'att-rate-bad-header',
      fetchFn: rateLimitedUpstream('3601'),
    })
    // Witness: the 429 path ran.
    expect(res.status).toBe(429)
    expect(res.body).toEqual({ error: 'rate_limited' })
    expect(res.headers['retry-after']).toBeUndefined()
  })

  // G1-3 (#720): an upstream refusal the same request would get again is a
  // non-retryable 422 upstream_rejected, not a retryable 503. 502 stays the
  // gateway's "nothing answered" signal (review R1-H2).
  it('(g1-3a) answers 422 upstream_rejected with the upstream status', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-rejected-http',
      fetchFn: (async () => new Response('not found', { status: 404 })) as typeof fetch,
    })
    expect(res.status).toBe(422)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'upstream_rejected', upstreamStatus: 404 })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-rejected-http',
      outcome: 'failed',
      code: 'upstream_rejected',
      details: { upstreamStatus: 404 },
      deliveredAs: 'http_status',
      httpStatus: 422,
    })
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'upstream_rejected')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  it('(g1-3c) carries the upstream status on the SSE error frame after a keepalive', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-rejected-sse',
      fetchFn: slowFailingUpstream(60, 402),
      configOverrides: { heartbeatIntervalMs: 20 },
    })
    expect(res.status).toBe(200)
    // Witness: a keepalive went out first, so only the frame can carry it.
    expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
    expect(
      res.text.endsWith(
        'data: {"type":"error","code":"upstream_rejected","upstreamStatus":402}\n\n'
      )
    ).toBe(true)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      code: 'upstream_rejected',
      details: { upstreamStatus: 402 },
      deliveredAs: 'sse_error',
    })
  })

  it('(g1-1d) delivers an upstream 429 after a keepalive as an SSE rate_limited frame', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-rate-sse',
      fetchFn: (async () => {
        await new Promise(resolve => setTimeout(resolve, 60))
        return new Response('slow down', { status: 429, headers: { 'retry-after': '7' } })
      }) as typeof fetch,
      configOverrides: { heartbeatIntervalMs: 20 },
    })
    expect(res.status).toBe(200)
    // Witness: a keepalive went out first, so the 429 can only travel as a frame.
    expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
    // The frame carries the code alone: no Retry-After, no upstream status.
    expect(res.text.endsWith('data: {"type":"error","code":"rate_limited"}\n\n')).toBe(true)
    expect(res.headers['retry-after']).toBeUndefined()
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      code: 'rate_limited',
      details: { retryAfterSeconds: 7 },
      deliveredAs: 'sse_error',
    })
  })

  // G1-4 (#720): a redeem whose connection nothing accepted (control-api or
  // its gateway restarting) is reported as the control plane being down.
  it('(g1-4a) answers 503 control_plane_unavailable when redeem cannot connect', async () => {
    const closed = createServer()
    await new Promise<void>(resolve => closed.listen(0, '127.0.0.1', () => resolve()))
    const { port } = closed.address() as AddressInfo
    await new Promise<void>(resolve => closed.close(() => resolve()))
    const upstreamFetch = vi.fn(upstream(1, 0))
    const { res, lines, unreachable, causeCodeLines, metricsText } = await run({
      providerAttemptId: 'att-control-down',
      fetchFn: upstreamFetch as unknown as typeof fetch,
      controlApiClient: new ControlApiClient({
        baseUrl: `http://127.0.0.1:${port}/api/v1`,
        serviceName: 'grok-llm-proxy',
        serviceToken: 'dev-grok-llm-proxy-token',
      }),
    })
    expect(res.status).toBe(503)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'control_plane_unavailable' })
    // Redeem failed, so the upstream was never called.
    expect(upstreamFetch).not.toHaveBeenCalled()
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-control-down',
      outcome: 'failed',
      code: 'control_plane_unavailable',
      causeCode: 'ECONNREFUSED',
      deliveredAs: 'http_status',
      httpStatus: 503,
    })
    expectNoForbiddenKeys(lines[0]!)
    // Review R1-M2: the hop line counts the failure; the cause code is on the
    // attempt line only, so one failure never reads as two.
    expect(unreachable).toHaveLength(1)
    expect('causeCode' in unreachable[0]!).toBe(false)
    expect(causeCodeLines).toEqual([lines[0]])
    expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  it('(a) answers 422 and logs one attempt line when 257 calls arrive before any text', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-limit-http',
      calls: 257,
    })
    expect(res.status).toBe(422)
    // The staged SSE headers are replaced by a JSON response.
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.headers['cache-control']).toBeUndefined()
    expect(res.body).toEqual({ error: 'tool_call_limit_exceeded' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-limit-http',
      outcome: 'failed',
      code: 'tool_call_limit_exceeded',
      reason: 'tool calls exceed 256',
      details: { limit: 256, observed: 257 },
      deliveredAs: 'http_status',
      httpStatus: 422,
      toolCalls: 0,
      textChunks: 0,
    })
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(1)
  })

  // A call count within `maxToolCalls` says nothing about the size of each
  // call. This refusal has to reach the Host as a 422 with its own code: a 503
  // would be classified as an overload and retried with the same oversized
  // response.
  it('(a1) answers 422 when one call’s arguments exceed the retained budget', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-args-http',
      fetchFn: oversizedArgumentsUpstream(),
    })
    expect(res.status).toBe(422)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'tool_call_arguments_exceeded' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-args-http',
      outcome: 'failed',
      code: 'tool_call_arguments_exceeded',
      details: {
        limit: MAX_TOOL_CALL_ARGUMENT_BYTES,
        observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_OVERRUN_CHUNK,
      },
      deliveredAs: 'http_status',
      httpStatus: 422,
      toolCalls: 0,
      textChunks: 0,
    })
    // The refusal text names the bound, never the arguments that tripped it.
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'tool_call_arguments_exceeded')).toBe(1)
  })

  // Arguments that are not a JSON object are invalid model output. The refusal
  // has to reach the Host as a 422 with its own code and metric label: the 503
  // default would be classified as an overload and retried.
  it('(a1b) answers 422 invalid_tool_arguments when a call’s arguments are not a JSON object', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-bad-args-http',
      fetchFn: malformedArgumentsUpstream(),
    })
    expect(res.status).toBe(422)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'invalid_tool_arguments' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-bad-args-http',
      outcome: 'failed',
      code: 'invalid_tool_arguments',
      deliveredAs: 'http_status',
      httpStatus: 422,
      toolCalls: 0,
      textChunks: 0,
    })
    expectNoForbiddenKeys(lines[0]!)
    // The attempt line names the refusal, never the arguments that caused it.
    expect(JSON.stringify(lines[0])).not.toContain('{\\"q\\":')
    expect(failureCount(metricsText, 'invalid_tool_arguments')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  // R10 (M1): the upstream context overflow reaches the Host as a 400 with its
  // own code and metric label. The 503 default would be classified as an
  // overload and retried with the identical conversation.
  it('(a1c) T-R10-2 answers 400 context_length_exceeded for the upstream context overflow', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-context-http',
      fetchFn: contextOverflowUpstream(),
    })
    expect(res.status).toBe(400)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'context_length_exceeded' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-context-http',
      outcome: 'failed',
      code: 'context_length_exceeded',
      deliveredAs: 'http_status',
      httpStatus: 400,
      toolCalls: 0,
      textChunks: 0,
    })
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'context_length_exceeded')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  it('(a2) sends an SSE error frame when text was already streamed', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-limit-sse',
      textDeltas: 1,
      calls: 257,
    })
    expect(res.status).toBe(200)
    expect(res.text).toContain('data: {"type":"text","text":"t0"}')
    expect(res.text).toContain('data: {"type":"error","code":"tool_call_limit_exceeded"}')
    expect(res.text).not.toContain('"type":"tool_call"')
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      outcome: 'failed',
      code: 'tool_call_limit_exceeded',
      deliveredAs: 'sse_error',
      textChunks: 1,
      toolCalls: 0,
    })
    expect('httpStatus' in lines[0]!).toBe(false)
    expectNoForbiddenKeys(lines[0]!)
  })

  it('(b) logs the counted frames of a successful attempt', async () => {
    const { res, lines, metricsText } = await run({
      providerAttemptId: 'att-success',
      calls: 3,
    })
    expect(res.status).toBe(200)
    expect(res.text).toContain('"type":"done","outcome":"success"')
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-success',
      outcome: 'success',
      deliveredAs: 'sse_done',
      toolCalls: 3,
      textChunks: 0,
    })
    expectNoForbiddenKeys(lines[0]!)
    // Witness: the attempt was counted, so the absent failure sample is meaningful.
    expect(metricsText).toMatch(
      /grok_llm_proxy_attempts_total\{outcome="success",operation="completion_stream"\} 1/
    )
    expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(0)
  })

  it('(c) serves a post-staging invalid request as JSON without the parse message', async () => {
    const secretField = 'sk-live-0123456789abcdef'
    const { res, lines } = await run({
      providerAttemptId: 'att-invalid',
      tamper: raw => {
        ;(raw.messages as Record<string, unknown>[])[0]![secretField] = 'x'
      },
    })
    // This error is raised inside the transport, after the SSE headers were
    // staged. The route's four earlier rejections run before that staging and
    // already answer JSON, so only this one proves the repair.
    expect(res.status).toBe(400)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.headers['cache-control']).toBeUndefined()
    expect(res.body).toEqual({ error: 'invalid_request' })
    // Witness: the attempt line is still emitted for the rejected request.
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-invalid',
      outcome: 'failed',
      code: 'invalid_request',
      deliveredAs: 'http_status',
      httpStatus: 400,
    })
    // The parser's message names caller-supplied fields, so it is withheld.
    expect('reason' in lines[0]!).toBe(false)
    expect(JSON.stringify(lines[0])).not.toContain(secretField)
    expectNoForbiddenKeys(lines[0]!)
  })

  it('(t1) answers 504 stream_duration_exceeded when the total cap fires before any frame', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-total-http',
      fetchFn: stalledUpstream(0),
      maxStreamDurationMs: 100,
    })
    expect(res.status).toBe(504)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'stream_duration_exceeded' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-total-http',
      outcome: 'failed',
      code: 'stream_duration_exceeded',
      reason: 'upstream stream exceeded maxStreamDurationMs',
      details: { limitMs: 100 },
      deliveredAs: 'http_status',
      httpStatus: 504,
    })
    expectNoForbiddenKeys(lines[0]!)
    expect(failureCount(metricsText, 'stream_duration_exceeded')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
    expect(timeoutCount(metricsText, 'total')).toBe(1)
    expect(timeoutCount(metricsText, 'idle')).toBe(0)
  })

  it('(t2) sends a stream_duration_exceeded SSE frame when text was already streamed', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-total-sse',
      fetchFn: stalledUpstream(1),
      maxStreamDurationMs: 100,
    })
    expect(res.status).toBe(200)
    expect(res.text).toContain('data: {"type":"text","text":"t0"}')
    expect(res.text).toContain('data: {"type":"error","code":"stream_duration_exceeded"}')
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      outcome: 'failed',
      code: 'stream_duration_exceeded',
      deliveredAs: 'sse_error',
      textChunks: 1,
    })
    expect('httpStatus' in lines[0]!).toBe(false)
  })

  it('(t3) answers 503 provider_unavailable and counts an idle timeout on a silent upstream', async () => {
    const { res, receipts, lines, metricsText } = await run({
      providerAttemptId: 'att-idle-http',
      fetchFn: stalledUpstream(0),
      configOverrides: { upstreamIdleTimeoutMs: 50 },
    })
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'provider_unavailable' })
    expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-idle-http',
      outcome: 'failed',
      code: 'provider_unavailable',
      reason: 'upstream stream idle timeout',
      details: { idleTimeoutMs: 50 },
      deliveredAs: 'http_status',
      httpStatus: 503,
    })
    expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
    expect(timeoutCount(metricsText, 'idle')).toBe(1)
    expect(timeoutCount(metricsText, 'total')).toBe(0)
  })

  it('(d) labels the failure metric with a known transport code', async () => {
    const { res, lines, metricsText } = await run({
      providerAttemptId: 'att-buffer',
      fetchFn: unterminatedUpstream(),
    })
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'sse_buffer_exceeded' })
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      outcome: 'failed',
      code: 'sse_buffer_exceeded',
      httpStatus: 503,
    })
    expect(failureCount(metricsText, 'sse_buffer_exceeded')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  it('(e) labels an unknown control-api code as other and keeps the raw code in the log', async () => {
    const rawCode = 'k8s says: pod foo not found'
    const { res, lines, metricsText } = await run({
      providerAttemptId: 'att-unknown-code',
      deniedCode: rawCode,
    })
    expect(res.status).toBe(503)
    // Witness: the attempt failed and was logged with the code control-api sent.
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({ outcome: 'failed', code: rawCode, httpStatus: 503 })
    expect(failureCount(metricsText, 'other')).toBe(1)
    expect(metricsText).not.toContain(rawCode)
  })

  // G1-5 (#720): the rpc gateway answers a redeem it could not deliver with
  // JSON `control_plane_unavailable`; the proxy passes it through with its own
  // status and metric label.
  it('(g1-5) passes a control_plane_unavailable redeem denial through as 503 with its own label', async () => {
    const { res, lines, metricsText } = await run({
      providerAttemptId: 'att-cp-json',
      deniedCode: 'control_plane_unavailable',
    })
    expect(res.status).toBe(503)
    expect(res.body).toEqual({ error: 'control_plane_unavailable' })
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      outcome: 'failed',
      code: 'control_plane_unavailable',
      httpStatus: 503,
    })
    expect(failureCount(metricsText, 'control_plane_unavailable')).toBe(1)
    expect(failureCount(metricsText, 'other')).toBe(0)
  })

  // `ATTEMPT_ERROR_STATUS` is an object literal, so an inherited name resolves
  // to a function or an object instead of undefined. A plain index read would
  // hand that value to `res.status()`, which throws ERR_HTTP_INVALID_STATUS_CODE
  // inside the request IIFE's catch — an unhandled rejection that takes every
  // other in-flight stream down with the process.
  it.each(['constructor', '__proto__', 'toString', 'hasOwnProperty'])(
    '(f) answers 503 for the inherited control-api code %j instead of crashing',
    async code => {
      const { res, lines, metricsText } = await run({
        providerAttemptId: `att-proto-${code}`,
        deniedCode: code,
      })
      expect(res.status).toBe(503)
      expect(res.body).toEqual({ error: code })
      // Witness: the attempt was reached and logged, so the status above is the
      // mapped refusal and not a connection that never produced a response.
      expect(lines).toHaveLength(1)
      expect(lines[0]).toMatchObject({ outcome: 'failed', code, httpStatus: 503 })
      expect(failureCount(metricsText, 'other')).toBe(1)
      expect(failureCount(metricsText, code)).toBe(0)
    }
  )

  it('(rl1) logs the request-limit reason and labels its failure metric request_limit', async () => {
    const acquire = vi
      .spyOn(streamGate, 'acquire')
      .mockRejectedValueOnce(
        new RequestLimitError('stream queue is full', 'queue_full', 'proxy_capacity_exceeded')
      )
    try {
      const { res, receipts, lines, metricsText } = await run({
        providerAttemptId: 'att-queue-full',
      })
      // Witness: the refusal came from the stream gate this test replaced.
      expect(acquire).toHaveBeenCalledTimes(1)
      expect(res.status).toBe(503)
      expect(res.body).toEqual({ error: 'proxy_capacity_exceeded' })
      // The gate refused before the redeem, so there is no receipt to finalize.
      expect(receipts).toEqual([])
      expect(lines).toHaveLength(1)
      expect(lines[0]).toMatchObject({
        providerAttemptId: 'att-queue-full',
        outcome: 'failed',
        code: 'proxy_capacity_exceeded',
        reason: 'stream queue is full',
        deliveredAs: 'http_status',
        httpStatus: 503,
      })
      expectNoForbiddenKeys(lines[0]!)
      expect(failureCount(metricsText, 'request_limit')).toBe(1)
      expect(failureCount(metricsText, 'provider_unavailable')).toBe(0)
    } finally {
      acquire.mockRestore()
    }
  })

  it('(hb1) keeps a silent upstream attempt open with SSE comments and counts them', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-heartbeat',
      fetchFn: silentThenCompletingUpstream(100),
      configOverrides: { heartbeatIntervalMs: 20 },
    })
    expect(res.status).toBe(200)
    const firstData = res.text.indexOf('data:')
    expect(firstData).toBeGreaterThan(0)
    expect(keepaliveCount(res.text.slice(0, firstData))).toBeGreaterThanOrEqual(2)
    expect(res.text).toContain('data: {"type":"text","text":"t0"}')
    // Nothing is written after the done frame.
    expect(res.text.endsWith('data: {"type":"done","outcome":"success"}\n\n')).toBe(true)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      providerAttemptId: 'att-heartbeat',
      outcome: 'success',
      deliveredAs: 'sse_done',
      heartbeats: keepaliveCount(res.text),
    })
  })

  it('(hb2) sends no keepalive before the redeem succeeds, so a denial keeps its HTTP status', async () => {
    const denied = slowDenyingClient('no_grant', 60)
    const { res, lines } = await run({
      providerAttemptId: 'att-heartbeat-denied',
      controlApiClient: denied.client,
      configOverrides: { heartbeatIntervalMs: 20 },
    })
    // Witness: the redeem ran and outlasted three heartbeat intervals.
    expect(denied.redeemCalls()).toBe(1)
    expect(res.status).toBe(403)
    expect(res.headers['content-type']).toMatch(/^application\/json/)
    expect(res.body).toEqual({ error: 'no_grant' })
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      code: 'no_grant',
      deliveredAs: 'http_status',
      httpStatus: 403,
      heartbeats: 0,
    })
  })

  it('(hb3) delivers an upstream failure after a keepalive as an SSE error frame', async () => {
    const { res, lines } = await run({
      providerAttemptId: 'att-heartbeat-failed',
      fetchFn: slowFailingUpstream(60, 500),
      configOverrides: { heartbeatIntervalMs: 20 },
    })
    expect(res.status).toBe(200)
    expect(keepaliveCount(res.text)).toBeGreaterThanOrEqual(1)
    expect(res.text.endsWith('data: {"type":"error","code":"provider_unavailable"}\n\n')).toBe(true)
    expect(lines).toHaveLength(1)
    expect(lines[0]).toMatchObject({
      outcome: 'failed',
      code: 'provider_unavailable',
      deliveredAs: 'sse_error',
      heartbeats: keepaliveCount(res.text),
    })
    expect('httpStatus' in lines[0]!).toBe(false)
  })

  // Node drops a write on a closed response without an error, so a heartbeat
  // the server never stops is invisible on the wire: it is an interval that
  // outlives its attempt. The test tracks the timers themselves.
  it('(hb4) clears every heartbeat timer once the attempt ends and keeps serving', async () => {
    const uncaught: unknown[] = []
    const onUncaught = (err: unknown): void => {
      uncaught.push(err)
    }
    const realSetInterval = globalThis.setInterval
    const heartbeatTimers: unknown[] = []
    const setSpy = vi.spyOn(globalThis, 'setInterval').mockImplementation(((
      handler: () => void,
      ms?: number
    ) => {
      const timer = realSetInterval(handler, ms)
      // 5 ms is the interval this test configures; nothing else uses it.
      if (ms === 5) heartbeatTimers.push(timer)
      return timer
    }) as unknown as typeof setInterval)
    const clearSpy = vi.spyOn(globalThis, 'clearInterval')
    process.on('uncaughtException', onUncaught)
    try {
      const first = await run({
        providerAttemptId: 'att-heartbeat-end',
        fetchFn: silentThenCompletingUpstream(30),
        configOverrides: { heartbeatIntervalMs: 5 },
      })
      // Witness: the heartbeat was running while the upstream was silent.
      expect(keepaliveCount(first.res.text)).toBeGreaterThanOrEqual(1)
      await new Promise(resolve => setTimeout(resolve, 50))
      expect(uncaught).toEqual([])
      const second = await run({
        providerAttemptId: 'att-heartbeat-next',
        textDeltas: 1,
        configOverrides: { heartbeatIntervalMs: 5 },
      })
      expect(second.res.status).toBe(200)
      expect(second.res.text).toContain('data: {"type":"done","outcome":"success"}')
      // Witness: each attempt started its own heartbeat.
      expect(heartbeatTimers).toHaveLength(2)
      const cleared = clearSpy.mock.calls.map(call => call[0])
      for (const timer of heartbeatTimers) expect(cleared).toContain(timer)
    } finally {
      process.off('uncaughtException', onUncaught)
      setSpy.mockRestore()
      clearSpy.mockRestore()
    }
  })
})

/**
 * #739 D1-bis — the stream-gate wait also ends when the execution ticket dies.
 * A request still queued at the ticket's `exp` could only be redeemed into a
 * certain `ticket_expired`, so it is answered as a capacity refusal instead,
 * with no redeem. The margin is zero: a request whose ticket is still alive
 * when a slot frees is served exactly as before.
 */
describe('grok-llm-proxy ticket-aware stream-gate wait (#739 D1-bis)', () => {
  /** Captured before any test fakes the clock, so waits stay on real time. */
  const realSetTimeout = globalThis.setTimeout
  const realSleep = (ms: number) => new Promise<void>(resolve => realSetTimeout(resolve, ms))
  const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
  const COMPLETIONS = '/internal/runtime/v1/grok/completions'

  async function until(condition: () => boolean, what: string): Promise<void> {
    const deadline = performance.now() + 5_000
    while (!condition()) {
      if (performance.now() > deadline) throw new Error(`${what} did not happen within 5 s`)
      await realSleep(10)
    }
  }

  function withinReal<T>(promise: Promise<T>, ms: number): Promise<T | undefined> {
    return Promise.race([promise, realSleep(ms).then(() => undefined)])
  }

  /** An envelope whose ticket expires exactly `ticketLifeMs` from the (fake) now. */
  function envelope(providerAttemptId: string, ticketLifeMs: number): Record<string, unknown> {
    const raw: Record<string, unknown> = {
      schemaVersion: 'grok-completion-request.v1',
      requestId: `req-${providerAttemptId}`,
      idempotencyKey: `idem-${providerAttemptId}`,
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'hello' }],
    }
    const parsed = parseGrokCompletionRequestV1(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    const executionTicket = jwt.sign(
      {
        jti: '77777777-7777-4777-8777-777777777777',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId,
        exp: (Date.now() + ticketLifeMs) / 1000,
      },
      privateKey,
      { algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
    )
    return { executionTicket, requestHash, request: raw }
  }

  function countingClient(redeemAnswer: 'granted' | 'ticket_expired'): {
    client: ControlApiClient
    redeems: () => number
  } {
    let redeems = 0
    const client = {
      async redeem(): Promise<RedeemAttemptSuccess> {
        redeems += 1
        if (redeemAnswer === 'ticket_expired') {
          throw new ControlApiClientError('ticket_expired', 'control API request denied')
        }
        return {
          accessToken: 'test-access-ticket-life',
          transport: {
            protocolVersion: 'grok-subscription-transport.v1',
            completionsOrigin: GROK_COMPLETIONS_ORIGIN,
            catalogOrigin: GROK_CATALOG_ORIGIN,
            operation: 'completion_stream',
            servedModel: 'gpt-5.1',
            maxStreamDurationMs: 1_800_000,
          },
          expiryClass: 'short_lived',
          attemptReceipt: 'd'.repeat(64),
        }
      },
      async finalize(input: {
        receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
      }): Promise<FinalizeAttemptSuccess> {
        return {
          providerAttemptId: input.receipt.providerAttemptId,
          outcome: input.receipt.outcome,
          duplicate: false,
        }
      },
    } as unknown as ControlApiClient
    return { client, redeems: () => redeems }
  }

  const upstream = (async () =>
    new Response(
      'data: {"type":"response.output_text.delta","delta":"t0"}\n\n' +
        'data: {"type":"response.completed"}\n\n',
      { status: 200, headers: { 'content-type': 'text/event-stream' } }
    )) as typeof fetch

  function failureCount(metricsText: string, code: string): number {
    const line = metricsText
      .split('\n')
      .find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))
    return line ? Number(line.split(' ').pop()) : 0
  }

  /** Fakes the clock on a whole second, so a ticket's `exp` lands on an exact instant. */
  function fakeClock(): void {
    vi.useFakeTimers({
      now: Math.ceil(Date.now() / 1000) * 1000,
      toFake: ['setTimeout', 'clearTimeout', 'Date'],
    })
  }

  /**
   * Frees the held slots and lets any waiter still polling the shared gate
   * finish on the fake clock, so the module gate is empty for the next test.
   */
  async function releaseAndDrain(slots: Array<() => void>): Promise<void> {
    for (const release of slots.splice(0)) release()
    await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)
    vi.useRealTimers()
  }

  async function saturate(): Promise<Array<() => void>> {
    const slots: Array<() => void> = []
    for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {
      slots.push(await streamGate.acquire())
    }
    return slots
  }

  it('T-AC-5-grok answers 503 provider_unavailable at the ticket expiry, without a redeem, while the gate stays saturated', async () => {
    fakeClock()
    const warn = vi.spyOn(logger, 'warn')
    let slots: Array<() => void> = []
    let acquire: ReturnType<typeof vi.spyOn> | undefined
    try {
      slots = await saturate()
      // Pass-through spy: it only counts the handler's calls into the gate.
      const spy = vi.spyOn(streamGate, 'acquire')
      acquire = spy
      const control = countingClient('ticket_expired')
      const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
        controlApiClient: control.client,
        fetchFn: upstream,
        lookup,
      })
      const ticketLifeMs = 20_000
      const reply = request(apps.runtimeApp)
        .post(COMPLETIONS)
        .set('Authorization', `Bearer ${platformToken()}`)
        .send(envelope('att-ticket-life', ticketLifeMs))
        .then(res => res)
      await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')

      await vi.advanceTimersByTimeAsync(ticketLifeMs - 1)
      // Witness: the request waited for the ticket's whole life, it was not
      // refused on arrival.
      expect(await withinReal(reply, 100)).toBeUndefined()
      await vi.advanceTimersByTimeAsync(1)
      const res = await withinReal(reply, 2_000)
      expect(res?.status).toBe(503)
      expect(res?.body).toEqual({ error: 'provider_unavailable' })

      // A slot that frees after the ticket died is never used to redeem it.
      slots.pop()?.()
      await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs)
      expect(control.redeems()).toBe(0)

      const refusals = warn.mock.calls
        .map(call => call[0] as unknown as Record<string, unknown>)
        .filter(entry => entry?.event === 'grok_proxy_admission_refused')
      expect(refusals).toEqual([
        {
          event: 'grok_proxy_admission_refused',
          reason: 'ticket_life',
          providerAttemptId: 'att-ticket-life',
          hostRef: 'research-host',
        },
      ])
      const metricsText = (await request(apps.probeApp).get('/metrics')).text
      expect(failureCount(metricsText, 'provider_unavailable')).toBe(1)
      expect(failureCount(metricsText, 'request_limit')).toBe(0)
    } finally {
      acquire?.mockRestore()
      await releaseAndDrain(slots)
      warn.mockRestore()
    }
  }, 30_000)

  it('T-PAR-TK-grok serves a request whose ticket is still alive when the gate frees at +50 s of a 60 s ticket', async () => {
    fakeClock()
    let slots: Array<() => void> = []
    let acquire: ReturnType<typeof vi.spyOn> | undefined
    try {
      slots = await saturate()
      const spy = vi.spyOn(streamGate, 'acquire')
      acquire = spy
      const control = countingClient('granted')
      const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
        controlApiClient: control.client,
        fetchFn: upstream,
        lookup,
      })
      const reply = request(apps.runtimeApp)
        .post(COMPLETIONS)
        .set('Authorization', `Bearer ${platformToken()}`)
        .send(envelope('att-ticket-alive', 60_000))
        .then(res => res)
      await until(() => spy.mock.calls.length === 1, 'the request queueing at the stream gate')

      await vi.advanceTimersByTimeAsync(50_000)
      expect(await withinReal(reply, 100)).toBeUndefined()
      slots.pop()?.()
      // One stream-gate poll interval.
      await vi.advanceTimersByTimeAsync(10)
      const res = await withinReal(reply, 2_000)
      expect(res?.status).toBe(200)
      expect(res?.text).toContain('data: {"type":"text","text":"t0"}')
      expect(res?.text).toContain('data: {"type":"done","outcome":"success"}')
      expect(control.redeems()).toBe(1)
    } finally {
      acquire?.mockRestore()
      await releaseAndDrain(slots)
    }
  }, 30_000)

  /** A V2 envelope past `maxBodyBytes`, so it takes the visual gate. */
  function visualEnvelope(
    providerAttemptId: string,
    ticketLifeMs: number
  ): Record<string, unknown> {
    const raw: Record<string, unknown> = {
      schemaVersion: 'grok-completion-request.v2',
      requestId: `req-${providerAttemptId}`,
      idempotencyKey: `idem-${providerAttemptId}`,
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'x'.repeat(80_000) }],
    }
    const parsed = parseGrokCompletionRequest(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    const executionTicket = jwt.sign(
      {
        jti: '78787878-7878-4787-8787-787878787878',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId,
        exp: (Date.now() + ticketLifeMs) / 1000,
      },
      privateKey,
      { algorithm: 'RS256', issuer: 'control-api', audience: 'grok-llm-proxy' }
    )
    return { executionTicket, requestHash, request: raw }
  }

  // R17-2 on the visual path: the visual gate is taken before the body is
  // parsed, so the ticket is read only once the slot frees. A ticket that died
  // during that wait reaches the ticket gate expired, and the answer is the
  // retryable ticket_expired, not the stream gate's ticket-life refusal.
  it('T-R17-2-visual-grok answers ticket_expired when the ticket died while the body waited at the visual gate', async () => {
    fakeClock()
    const warn = vi.spyOn(logger, 'warn')
    let held: (() => void) | undefined
    try {
      held = await visualStreamGate.acquire()
      const control = countingClient('granted')
      const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
        controlApiClient: control.client,
        fetchFn: upstream,
        lookup,
      })
      const ticketLifeMs = 20_000
      const body = visualEnvelope('att-visual-expired', ticketLifeMs)
      expect(Buffer.byteLength(JSON.stringify(body))).toBeGreaterThan(65_536)
      const reply = request(apps.runtimeApp)
        .post(COMPLETIONS)
        .set('Authorization', `Bearer ${platformToken()}`)
        .send(body)
        .then(res => res)
      await until(
        () => visualStreamGate.snapshot().queued === 1,
        'the body queueing at the visual gate'
      )

      await vi.advanceTimersByTimeAsync(ticketLifeMs)
      // Witness: the body was still waiting for the slot at the ticket's expiry.
      expect(await withinReal(reply, 100)).toBeUndefined()
      held()
      held = undefined
      // One stream-gate poll interval.
      await vi.advanceTimersByTimeAsync(10)
      const res = await withinReal(reply, 2_000)
      expect(res?.status).toBe(403)
      expect(res?.body).toEqual({ error: 'ticket_expired' })
      expect(control.redeems()).toBe(0)

      const events = warn.mock.calls.map(call => call[0] as unknown as Record<string, unknown>)
      expect(events.filter(entry => entry?.event === 'grok_proxy_denied')).toEqual([
        { event: 'grok_proxy_denied', code: 'ticket_expired' },
      ])
      expect(events.filter(entry => entry?.event === 'grok_proxy_admission_refused')).toEqual([])
      // The ticket refusal released the visual slot the body was parsed under.
      expect(visualStreamGate.snapshot()).toEqual({ running: 0, queued: 0 })
      expect(streamGate.snapshot()).toEqual({ running: 0, queued: 0 })
    } finally {
      held?.()
      vi.useRealTimers()
      warn.mockRestore()
    }
  }, 30_000)

  // The visual gate's own wait is the admission clock stamped at arrival. A
  // body that reaches the gate some ms after arrival and waits that clock out
  // is refused as the gate's own queue wait, `visual_gate`: the same answer as
  // a body that reached the gate in the arrival millisecond.
  it.each([0, 5])(
    'T-VG-QW-grok answers 503 visual_gate when a body enqueued %i ms after arrival waits out the admission clock',
    async lagMs => {
      fakeClock()
      const warn = vi.spyOn(logger, 'warn')
      const held: Array<() => void> = []
      let acquire: { mockRestore: () => void } | undefined
      try {
        for (let i = 0; i < VISUAL_STREAM_LIMITS.maxConcurrentStreams; i += 1) {
          held.push(await visualStreamGate.acquire())
        }
        const enqueue = visualStreamGate.acquire.bind(visualStreamGate)
        // Moves the clock `lagMs` between the arrival stamp and the enqueue,
        // then calls the real gate.
        const spy = vi
          .spyOn(visualStreamGate, 'acquire')
          .mockImplementation((signal, deadlineAt, queueWaitStartedAt) => {
            vi.setSystemTime(Date.now() + lagMs)
            return enqueue(signal, deadlineAt, queueWaitStartedAt)
          })
        acquire = spy
        const control = countingClient('granted')
        const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
          controlApiClient: control.client,
          fetchFn: upstream,
          lookup,
        })
        const arrivedAt = Date.now()
        const reply = request(apps.runtimeApp)
          .post(COMPLETIONS)
          .set('Authorization', `Bearer ${platformToken()}`)
          .send(visualEnvelope(`att-visual-wait-${lagMs}`, 60_000))
          .then(res => res)
        await until(
          () => spy.mock.calls.length === 1 && visualStreamGate.snapshot().queued === 1,
          'the body queueing at the visual gate'
        )
        // Witness: the gate was handed the admission clock and the arrival
        // instant, and it enqueued the body `lagMs` after arrival.
        expect(spy.mock.calls[0]?.slice(1)).toEqual([
          arrivedAt + STREAM_LIMITS.maxQueueWaitMs,
          arrivedAt,
        ])
        expect(Date.now()).toBe(arrivedAt + lagMs)

        await vi.advanceTimersByTimeAsync(STREAM_LIMITS.maxQueueWaitMs - lagMs - 1)
        // Witness: still queued one millisecond before the admission clock
        // runs out, so the body was not refused early.
        expect(await withinReal(reply, 100)).toBeUndefined()
        expect(visualStreamGate.snapshot().queued).toBe(1)
        await vi.advanceTimersByTimeAsync(1)
        const res = await withinReal(reply, 2_000)
        expect(Date.now()).toBe(arrivedAt + STREAM_LIMITS.maxQueueWaitMs)
        expect(res?.status).toBe(503)
        expect(res?.body).toEqual({ error: 'visual_gate' })
        expect(control.redeems()).toBe(0)
        const refusals = warn.mock.calls
          .map(call => call[0] as unknown as Record<string, unknown>)
          .filter(entry => entry?.event === 'grok_proxy_admission_refused')
        expect(refusals).toEqual([
          {
            event: 'grok_proxy_admission_refused',
            reason: 'visual_gate',
            code: 'visual_gate',
            detail: 'stream queue wait exceeded',
          },
        ])
        expect(visualStreamGate.snapshot()).toEqual({ running: held.length, queued: 0 })
      } finally {
        acquire?.mockRestore()
        for (const release of held.splice(0)) release()
        vi.useRealTimers()
        warn.mockRestore()
      }
    },
    30_000
  )
})

/**
 * #739 D2 — a body's budget reservation covers the phase in which several
 * copies of it are alive: reading, parsing, hashing and forwarding. That phase
 * ends when the upstream fetch resolves, because the whole request body has
 * been written by then. The reservation is released there instead of when the
 * SSE stream closes; the response's `close` event still releases it on every
 * path that never reaches the upstream.
 */
describe('grok-llm-proxy body budget release on upstream acceptance (#739 D2)', () => {
  const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
  const COMPLETIONS = '/internal/runtime/v1/grok/completions'
  const sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))

  async function until(condition: () => boolean, what: string): Promise<void> {
    const deadline = performance.now() + 10_000
    while (!condition()) {
      if (performance.now() > deadline) throw new Error(`${what} did not happen within 10 s`)
      await sleep(10)
    }
  }

  /** A runtime envelope with one user message of `contentChars` and a 60 s ticket. */
  function completionBody(
    contentChars: number,
    providerAttemptId: string,
    executionTicket?: string
  ): string {
    const raw = {
      schemaVersion: 'grok-completion-request.v1',
      requestId: `req-${providerAttemptId}`,
      idempotencyKey: `idem-${providerAttemptId}`,
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'x'.repeat(contentChars) }],
    }
    const parsed = parseGrokCompletionRequestV1(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    return JSON.stringify({
      executionTicket:
        executionTicket ??
        sign(
          {
            jti: '88888888-8888-4888-8888-888888888888',
            typ: 'grok-execution-ticket',
            hostRef: 'research-host',
            model: 'gpt-5.1',
            requestHash,
            providerAttemptId,
          },
          'grok-llm-proxy'
        ),
      requestHash,
      request: raw,
    })
  }

  function controlClient(redeemAnswer: 'granted' | 'ticket_expired'): {
    client: ControlApiClient
    redeems: () => number
  } {
    let redeems = 0
    const client = {
      async redeem(): Promise<RedeemAttemptSuccess> {
        redeems += 1
        if (redeemAnswer === 'ticket_expired') {
          throw new ControlApiClientError('ticket_expired', 'control API request denied')
        }
        return {
          accessToken: 'test-access-body-release',
          transport: {
            protocolVersion: 'grok-subscription-transport.v1',
            completionsOrigin: GROK_COMPLETIONS_ORIGIN,
            catalogOrigin: GROK_CATALOG_ORIGIN,
            operation: 'completion_stream',
            servedModel: 'gpt-5.1',
            maxStreamDurationMs: 1_800_000,
          },
          expiryClass: 'short_lived',
          attemptReceipt: 'e'.repeat(64),
        }
      },
      async finalize(input: {
        receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
      }): Promise<FinalizeAttemptSuccess> {
        return {
          providerAttemptId: input.receipt.providerAttemptId,
          outcome: input.receipt.outcome,
          duplicate: false,
        }
      },
    } as unknown as ControlApiClient
    return { client, redeems: () => redeems }
  }

  type UpstreamProbe = {
    fetchFn: typeof fetch
    calls: () => number
    /** True once the signal the proxy gave the upstream fetch has aborted. */
    aborted: () => boolean
  }

  function upstreamProbe(answer: (signal: AbortSignal) => Promise<Response>): UpstreamProbe {
    let calls = 0
    let seen: AbortSignal | undefined
    const fetchFn = (async (_url: unknown, init?: RequestInit) => {
      calls += 1
      const signal = init?.signal
      if (!signal) throw new Error('the upstream fetch carried no abort signal')
      seen = signal
      return answer(signal)
    }) as typeof fetch
    return { fetchFn, calls: () => calls, aborted: () => seen?.aborted === true }
  }

  /** Answers headers and one text delta, then stays open until the fetch aborts. */
  function openStreamUpstream(): UpstreamProbe {
    return upstreamProbe(
      async signal =>
        new Response(
          new ReadableStream<Uint8Array>({
            start(controller) {
              controller.enqueue(
                new TextEncoder().encode(
                  'data: {"type":"response.output_text.delta","delta":"t0"}\n\n'
                )
              )
              signal.addEventListener('abort', () => controller.error(signal.reason), {
                once: true,
              })
            },
          }),
          { status: 200, headers: { 'content-type': 'text/event-stream' } }
        )
    )
  }

  /** Never answers: the fetch stays pending until it aborts. */
  function pendingUpstream(): UpstreamProbe {
    return upstreamProbe(
      signal =>
        new Promise<Response>((_resolve, reject) => {
          signal.addEventListener('abort', () => reject(signal.reason), { once: true })
        })
    )
  }

  /**
   * The runtime app on a real listener, so an SSE stream can be observed while
   * it is still open. The body budget is private to `createProxyApps`; a
   * pass-through spy on `BodyBudget.prototype.acquire` records the instance.
   */
  async function listeningProxy(client: ControlApiClient, fetchFn: typeof fetch) {
    const acquire = vi.spyOn(BodyBudget.prototype, 'acquire')
    const apps = createProxyApps(config({ maxBodyBytes: DEFAULT_MAX_BODY_BYTES }), {
      controlApiClient: client,
      fetchFn,
      lookup,
    })
    let closes = 0
    apps.runtime.on('request', (_req, res) => {
      res.once('close', () => {
        closes += 1
      })
    })
    await new Promise<void>(resolve => apps.runtime.listen(0, '127.0.0.1', () => resolve()))
    const { port } = apps.runtime.address() as AddressInfo
    return {
      port,
      /** Declared sizes the body budget was asked for, in call order. */
      reservations: () => acquire.mock.calls.map(call => call[0]),
      budget: (): BodyBudget => {
        const instance = acquire.mock.contexts[0]
        if (!(instance instanceof BodyBudget)) throw new Error('no body reached the body budget')
        return instance
      },
      /** Responses whose `close` event fired. */
      closes: () => closes,
      close: async () => {
        acquire.mockRestore()
        await apps.close()
      },
    }
  }

  /** POSTs `payload` with a declared length and records the reply as it arrives. */
  function open(port: number, payload: string) {
    let status: number | undefined
    let received = ''
    let ended = false
    const errors: Error[] = []
    const req = httpRequest(
      {
        host: '127.0.0.1',
        port,
        method: 'POST',
        path: COMPLETIONS,
        agent: false,
        headers: {
          'content-type': 'application/json',
          authorization: `Bearer ${platformToken()}`,
          'content-length': Buffer.byteLength(payload),
        },
      },
      res => {
        status = res.statusCode
        res.setEncoding('utf8')
        res.on('data', (chunk: string) => {
          received += chunk
        })
        res.on('end', () => {
          ended = true
        })
      }
    )
    // Recorded, not ignored: a test that destroys the request expects one.
    req.on('error', err => errors.push(err))
    req.end(payload)
    return {
      status: () => status,
      received: () => received,
      ended: () => ended,
      errors: () => errors,
      destroy: () => req.destroy(),
    }
  }

  it('T-BR-1-grok releases the reservation of a near-cap body once the upstream accepted it, while the SSE stream is still open', async () => {
    const control = controlClient('granted')
    const upstream = openStreamUpstream()
    const proxy = await listeningProxy(control.client, upstream.fetchFn)
    const payload = completionBody(Math.floor(LIMITS.maxRequestBodyBytes * 0.99), 'att-br-1')
    const bytes = Buffer.byteLength(payload)
    // Fixture check: the body is near the 8 MiB cap and still admitted by it.
    expect(bytes).toBeGreaterThan(LIMITS.maxRequestBodyBytes * 0.99)
    expect(bytes).toBeLessThanOrEqual(DEFAULT_MAX_BODY_BYTES)
    const client = open(proxy.port, payload)
    try {
      // Witness: the first frame reached the client, so the upstream accepted
      // the request and the stream is live.
      await until(
        () => client.received().includes('data: {"type":"text","text":"t0"}'),
        'the first SSE frame'
      )
      expect(client.status()).toBe(200)
      expect(client.ended()).toBe(false)
      expect(upstream.calls()).toBe(1)
      expect(control.redeems()).toBe(1)
      expect(proxy.reservations()).toEqual([bytes])
      expect(proxy.budget().inFlightBytes).toBe(0)
    } finally {
      client.destroy()
      await proxy.close()
    }
  }, 30_000)

  it('T-BR-2-grok keeps the reservation while the upstream fetch is pending and releases it when the client leaves', async () => {
    const control = controlClient('granted')
    const upstream = pendingUpstream()
    const proxy = await listeningProxy(control.client, upstream.fetchFn)
    const payload = completionBody(64, 'att-br-2')
    const bytes = Buffer.byteLength(payload)
    const client = open(proxy.port, payload)
    try {
      // Witness: the attempt was redeemed and its upstream fetch started.
      await until(() => upstream.calls() === 1, 'the upstream fetch')
      expect(control.redeems()).toBe(1)
      expect(proxy.reservations()).toEqual([bytes])
      // Not released early: the fetch has not resolved, so the body may still
      // be being written.
      expect(proxy.budget().inFlightBytes).toBe(bytes)

      client.destroy()
      await until(() => upstream.aborted(), 'the client abort reaching the upstream fetch')
      await until(() => proxy.budget().inFlightBytes === 0, 'the reservation release')
      expect(client.status()).toBeUndefined()
    } finally {
      client.destroy()
      await proxy.close()
    }
  }, 30_000)

  it('T-BR-3a-grok releases the reservation of a request refused 403 ticket_invalid when its response closes', async () => {
    const control = controlClient('granted')
    const upstream = openStreamUpstream()
    const proxy = await listeningProxy(control.client, upstream.fetchFn)
    const payload = completionBody(64, 'att-br-3a', 'not-a-signed-ticket')
    const client = open(proxy.port, payload)
    try {
      await until(() => client.ended(), 'the refusal')
      expect(client.status()).toBe(403)
      expect(JSON.parse(client.received())).toEqual({ error: 'ticket_invalid' })
      await until(() => proxy.closes() === 1, "the response's close event")
      expect(client.errors()).toEqual([])
      expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
      expect(control.redeems()).toBe(0)
      expect(proxy.budget().inFlightBytes).toBe(0)
    } finally {
      await proxy.close()
    }
  }, 30_000)

  it('T-BR-3b-grok releases the reservation of a request refused 503 by a full stream gate when its response closes', async () => {
    const control = controlClient('granted')
    const upstream = openStreamUpstream()
    const proxy = await listeningProxy(control.client, upstream.fetchFn)
    const slots: Array<() => void> = []
    const queueAbort = new AbortController()
    const waiters: Array<Promise<() => void>> = []
    try {
      for (let i = 0; i < STREAM_LIMITS.maxConcurrentStreams; i += 1) {
        slots.push(await streamGate.acquire())
      }
      for (let i = 0; i < STREAM_LIMITS.maxQueuedRequests; i += 1) {
        waiters.push(streamGate.acquire(queueAbort.signal))
      }
      // Fixture check: the gate has no snapshot, so a probe shows the queue is full.
      await expect(streamGate.acquire()).rejects.toMatchObject({
        name: 'RequestLimitError',
        message: 'stream queue is full',
      })
      const payload = completionBody(64, 'att-br-3b')
      const client = open(proxy.port, payload)
      await until(() => client.ended(), 'the refusal')
      expect(client.status()).toBe(503)
      expect(JSON.parse(client.received())).toEqual({ error: 'proxy_capacity_exceeded' })
      await until(() => proxy.closes() === 1, "the response's close event")
      expect(client.errors()).toEqual([])
      expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
      expect(control.redeems()).toBe(0)
      expect(proxy.budget().inFlightBytes).toBe(0)
    } finally {
      queueAbort.abort()
      await Promise.allSettled(waiters)
      for (const release of slots.splice(0)) release()
      await proxy.close()
    }
  }, 30_000)

  it('T-BR-3c-grok releases the reservation of a request whose redeem failed when its response closes', async () => {
    const control = controlClient('ticket_expired')
    const upstream = openStreamUpstream()
    const proxy = await listeningProxy(control.client, upstream.fetchFn)
    const payload = completionBody(64, 'att-br-3c')
    const client = open(proxy.port, payload)
    try {
      await until(() => client.ended(), 'the refusal')
      expect(client.status()).toBe(403)
      expect(JSON.parse(client.received())).toEqual({ error: 'ticket_expired' })
      await until(() => proxy.closes() === 1, "the response's close event")
      expect(client.errors()).toEqual([])
      // Witness: the path reached the redeem and stopped before the upstream.
      expect(control.redeems()).toBe(1)
      expect(upstream.calls()).toBe(0)
      expect(proxy.reservations()).toEqual([Buffer.byteLength(payload)])
      expect(proxy.budget().inFlightBytes).toBe(0)
    } finally {
      await proxy.close()
    }
  }, 30_000)
})

/**
 * #739 D6 — on SIGTERM `main.ts` awaits `servers.close()`. The Deployment's
 * termination grace period (the stream cap plus 60 s) is only useful if that
 * close waits for an open SSE stream instead of cutting it.
 */
describe('grok-llm-proxy graceful drain on shutdown (#739 D6)', () => {
  const sleep = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms))

  async function until(condition: () => boolean, what: string): Promise<void> {
    const deadline = performance.now() + 10_000
    while (!condition()) {
      if (performance.now() > deadline) throw new Error(`${what} did not happen within 10 s`)
      await sleep(10)
    }
  }

  function completionBody(): string {
    const raw = {
      schemaVersion: 'grok-completion-request.v1',
      requestId: 'req-att-drain',
      idempotencyKey: 'idem-att-drain',
      provider: 'grok-subscription',
      model: 'gpt-5.1',
      messages: [{ role: 'user', content: 'hello' }],
    }
    const parsed = parseGrokCompletionRequestV1(raw)
    if (!parsed.ok) throw new Error(parsed.message)
    const requestHash = hashGrokCompletionRequest(parsed.value)
    const executionTicket = sign(
      {
        jti: '99999999-9999-4999-8999-999999999999',
        typ: 'grok-execution-ticket',
        hostRef: 'research-host',
        model: 'gpt-5.1',
        requestHash,
        providerAttemptId: 'att-drain',
      },
      'grok-llm-proxy'
    )
    return JSON.stringify({ executionTicket, requestHash, request: raw })
  }

  const client = {
    async redeem(): Promise<RedeemAttemptSuccess> {
      return {
        accessToken: 'test-access-drain',
        transport: {
          protocolVersion: 'grok-subscription-transport.v1',
          completionsOrigin: GROK_COMPLETIONS_ORIGIN,
          catalogOrigin: GROK_CATALOG_ORIGIN,
          operation: 'completion_stream',
          servedModel: 'gpt-5.1',
          maxStreamDurationMs: 1_800_000,
        },
        expiryClass: 'short_lived',
        attemptReceipt: 'f'.repeat(64),
      }
    },
    async finalize(input: {
      receipt: { providerAttemptId: string; outcome: FinalizeAttemptSuccess['outcome'] }
    }): Promise<FinalizeAttemptSuccess> {
      return {
        providerAttemptId: input.receipt.providerAttemptId,
        outcome: input.receipt.outcome,
        duplicate: false,
      }
    },
  } as unknown as ControlApiClient

  it('T-DR-1-grok close() does not resolve while an SSE stream is open and resolves after the stream ended', async () => {
    const encoder = new TextEncoder()
    let finishUpstream: (() => void) | undefined
    const fetchFn = (async () =>
      new Response(
        new ReadableStream<Uint8Array>({
          start(controller) {
            controller.enqueue(
              encoder.encode('data: {"type":"response.output_text.delta","delta":"t0"}\n\n')
            )
            finishUpstream = () => {
              controller.enqueue(encoder.encode('data: {"type":"response.completed"}\n\n'))
              controller.close()
            }
          },
        }),
        { status: 200, headers: { 'content-type': 'text/event-stream' } }
      )) as typeof fetch
    const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
      controlApiClient: client,
      fetchFn,
      lookup: async () => [{ address: '1.2.3.4', family: 4 }],
    })
    const order: string[] = []
    apps.runtime.on('request', (_req, res) => {
      res.once('finish', () => order.push('response finished'))
    })
    await new Promise<void>(resolve => apps.runtime.listen(0, '127.0.0.1', () => resolve()))
    const { port } = apps.runtime.address() as AddressInfo
    const payload = completionBody()
    let received = ''
    let ended = false
    const req = httpRequest(
      {
        host: '127.0.0.1',
        port,
        method: 'POST',
        path: '/internal/runtime/v1/grok/completions',
        agent: false,
        headers: {
          'content-type': 'application/json',
          authorization: `Bearer ${platformToken()}`,
          'content-length': Buffer.byteLength(payload),
        },
      },
      res => {
        res.setEncoding('utf8')
        res.on('data', (chunk: string) => {
          received += chunk
        })
        res.on('end', () => {
          ended = true
        })
      }
    )
    const errors: Error[] = []
    req.on('error', err => errors.push(err))
    req.end(payload)
    let closing: Promise<void> | undefined
    try {
      await until(
        () => received.includes('data: {"type":"text","text":"t0"}'),
        'the first SSE frame'
      )
      closing = apps.close().then(() => {
        order.push('closed')
      })
      await sleep(300)
      // Witness: the stream is still open, and close() is still waiting on it.
      expect(ended).toBe(false)
      expect(order).toEqual([])

      if (!finishUpstream) throw new Error('the upstream stream never started')
      finishUpstream()
      await until(() => order.includes('closed'), 'close() resolving')
      expect(order).toEqual(['response finished', 'closed'])
      await until(() => ended, 'the client seeing the end of the stream')
      expect(received).toContain('data: {"type":"done","outcome":"success"}')
      expect(errors).toEqual([])
    } finally {
      req.destroy()
      if (!closing) await apps.close()
      else await closing
    }
  }, 30_000)
})