Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(grok): cover the attempt line, the failure metric and the error body

Alfredo Lopez4e1a5a587b85
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 6e53b9461..3771605df 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import jwt from 'jsonwebtoken'
import { generateKeyPairSync } from 'node:crypto'
import request from 'supertest'
@@ -9,7 +9,14 @@ import {
import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'
import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'
import { type GrokLlmProxyConfig, loadConfig } from '../src/config.js'
-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'
+import {
+ ControlApiClient,
+ ControlApiClientError,
+ type FinalizeAttemptSuccess,
+ type RedeemAttemptSuccess,
+} from '../src/controlApiClient.js'
+import { logger } from '../src/logger.js'
+import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
import { createProxyApps } from '../src/server.js'
const { privateKey, publicKey } = generateKeyPairSync('rsa', {
@@ -445,3 +452,290 @@ describe('grok-llm-proxy startup config', () => {
}
)
})
+
+describe('grok-llm-proxy attempt telemetry', () => {
+ const lookup = async () => [{ address: '1.2.3.4', family: 4 }]
+ // Keys that carry caller content. The attempt line is identifiers and counts;
+ // none of these may ever reach it.
+ const FORBIDDEN_LOG_KEYS = ['body', 'request', 'executionTicket', 'arguments']
+
+ function completionBody(
+ providerAttemptId: string,
+ tamper?: (raw: Record<string, unknown>) => void
+ ): Record<string, unknown> {
+ const raw: Record<string, unknown> = {
+ schemaVersion: 'grok-completion-request.v1',
+ requestId: `req-${providerAttemptId}`,
+ idempotencyKey: `idem-${providerAttemptId}`,
+ provider: 'grok-subscription',
+ model: 'gpt-5.1',
+ messages: [{ role: 'user', content: 'hello' }],
+ }
+ const parsed = parseGrokCompletionRequestV1(raw)
+ if (!parsed.ok) throw new Error(parsed.message)
+ const requestHash = hashGrokCompletionRequestV1(parsed.value)
+ // Applied after hashing: the ticket stays bound to the untampered request,
+ // so the transport's parser rejects the body before any hash comparison.
+ tamper?.(raw)
+ const executionTicket = sign(
+ {
+ jti: '55555555-5555-4555-8555-555555555555',
+ typ: 'grok-execution-ticket',
+ hostRef: 'research-host',
+ model: 'gpt-5.1',
+ requestHash,
+ providerAttemptId,
+ },
+ 'grok-llm-proxy'
+ )
+ return { executionTicket, requestHash, request: raw }
+ }
+
+ function grantingClient(): { client: ControlApiClient; receipts: unknown[] } {
+ const receipts: unknown[] = []
+ const client = {
+ async redeem(): Promise<RedeemAttemptSuccess> {
+ return {
+ accessToken: 'test-access-telemetry',
+ transport: {
+ protocolVersion: 'grok-subscription-transport.v1',
+ completionsOrigin: GROK_COMPLETIONS_ORIGIN,
+ catalogOrigin: GROK_CATALOG_ORIGIN,
+ operation: 'completion_stream',
+ servedModel: 'gpt-5.1',
+ maxStreamDurationMs: 300_000,
+ },
+ expiryClass: 'short_lived',
+ attemptReceipt: 'c'.repeat(64),
+ }
+ },
+ async finalize(input: {
+ receipt: { outcome: FinalizeAttemptSuccess['outcome'] }
+ }): Promise<FinalizeAttemptSuccess> {
+ receipts.push(input.receipt)
+ return { providerAttemptId: 'att', outcome: input.receipt.outcome, duplicate: false }
+ },
+ } as unknown as ControlApiClient
+ return { client, receipts }
+ }
+
+ function upstream(textDeltas: number, calls: number): typeof fetch {
+ const frames: string[] = []
+ for (let index = 0; index < textDeltas; index += 1) {
+ frames.push(
+ `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${index}` })}\n\n`
+ )
+ }
+ for (let index = 0; index < calls; index += 1) {
+ frames.push(
+ `data: ${JSON.stringify({
+ type: 'response.output_item.done',
+ item: { type: 'function_call', id: `call-${index}`, name: 'lookup', arguments: '{}' },
+ })}\n\n`
+ )
+ }
+ frames.push('data: {"type":"response.completed"}\n\n')
+ return (async () =>
+ new Response(frames.join(''), {
+ status: 200,
+ headers: { 'content-type': 'text/event-stream' },
+ })) as typeof fetch
+ }
+
+ // One SSE block larger than the transport's 1 MiB buffer and never terminated,
+ // so the buffer guard fires before any frame is delivered.
+ function unterminatedUpstream(): typeof fetch {
+ return (async () =>
+ new Response(`data: ${'x'.repeat(1_100_000)}`, {
+ status: 200,
+ headers: { 'content-type': 'text/event-stream' },
+ })) as typeof fetch
+ }
+
+ function denyingClient(code: string): ControlApiClient {
+ return {
+ async redeem() {
+ throw new ControlApiClientError(code, 'control API request denied')
+ },
+ async finalize() {
+ throw new Error('finalize must not run after a denied redeem')
+ },
+ } as unknown as ControlApiClient
+ }
+
+ function failureCount(metricsText: string, code: string): number {
+ const line = metricsText
+ .split('\n')
+ .find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))
+ return line ? Number(line.split(' ').pop()) : 0
+ }
+
+ async function run(options: {
+ providerAttemptId: string
+ textDeltas?: number
+ calls?: number
+ tamper?: (raw: Record<string, unknown>) => void
+ deniedCode?: string
+ fetchFn?: typeof fetch
+ }) {
+ const info = vi.spyOn(logger, 'info')
+ const warn = vi.spyOn(logger, 'warn')
+ const { client, receipts } = grantingClient()
+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {
+ controlApiClient: options.deniedCode ? denyingClient(options.deniedCode) : client,
+ fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),
+ lookup,
+ })
+ try {
+ const res = await request(apps.runtimeApp)
+ .post('/internal/runtime/v1/grok/completions')
+ .set('Authorization', `Bearer ${platformToken()}`)
+ .send(completionBody(options.providerAttemptId, options.tamper))
+ const metricsText = (await request(apps.probeApp).get('/metrics')).text
+ const lines = [...info.mock.calls, ...warn.mock.calls]
+ .map(call => call[0] as unknown as Record<string, unknown>)
+ .filter(entry => entry?.event === 'grok_proxy_attempt_finished')
+ return { res, receipts, lines, metricsText }
+ } finally {
+ info.mockRestore()
+ warn.mockRestore()
+ }
+ }
+
+ function expectNoForbiddenKeys(line: Record<string, unknown>): void {
+ for (const key of FORBIDDEN_LOG_KEYS) expect(key in line).toBe(false)
+ }
+
+ it('(a) answers 422 and logs one attempt line when 65 calls arrive before any text', async () => {
+ const { res, receipts, lines, metricsText } = await run({
+ providerAttemptId: 'att-limit-http',
+ calls: 65,
+ })
+ expect(res.status).toBe(422)
+ // The staged SSE headers are replaced by a JSON response.
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.headers['cache-control']).toBeUndefined()
+ expect(res.body).toEqual({ error: 'tool_call_limit_exceeded' })
+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-limit-http',
+ outcome: 'failed',
+ code: 'tool_call_limit_exceeded',
+ reason: 'tool calls exceed 64',
+ details: { limit: 64, observed: 65 },
+ deliveredAs: 'http_status',
+ httpStatus: 422,
+ toolCalls: 0,
+ textChunks: 0,
+ })
+ expectNoForbiddenKeys(lines[0]!)
+ expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(1)
+ })
+
+ it('(a2) sends an SSE error frame when text was already streamed', async () => {
+ const { res, lines } = await run({
+ providerAttemptId: 'att-limit-sse',
+ textDeltas: 1,
+ calls: 65,
+ })
+ expect(res.status).toBe(200)
+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')
+ expect(res.text).toContain('data: {"type":"error","code":"tool_call_limit_exceeded"}')
+ expect(res.text).not.toContain('"type":"tool_call"')
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ outcome: 'failed',
+ code: 'tool_call_limit_exceeded',
+ deliveredAs: 'sse_error',
+ textChunks: 1,
+ toolCalls: 0,
+ })
+ expect('httpStatus' in lines[0]!).toBe(false)
+ expectNoForbiddenKeys(lines[0]!)
+ })
+
+ it('(b) logs the counted frames of a successful attempt', async () => {
+ const { res, lines, metricsText } = await run({
+ providerAttemptId: 'att-success',
+ calls: 3,
+ })
+ expect(res.status).toBe(200)
+ expect(res.text).toContain('"type":"done","outcome":"success"')
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-success',
+ outcome: 'success',
+ deliveredAs: 'sse_done',
+ toolCalls: 3,
+ textChunks: 0,
+ })
+ expectNoForbiddenKeys(lines[0]!)
+ // Witness: the attempt was counted, so the absent failure sample is meaningful.
+ expect(metricsText).toMatch(
+ /grok_llm_proxy_attempts_total\{outcome="success",operation="completion_stream"\} 1/
+ )
+ expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(0)
+ })
+
+ it('(c) serves a post-staging invalid request as JSON without the parse message', async () => {
+ const secretField = 'sk-live-0123456789abcdef'
+ const { res, lines } = await run({
+ providerAttemptId: 'att-invalid',
+ tamper: raw => {
+ ;(raw.messages as Record<string, unknown>[])[0]![secretField] = 'x'
+ },
+ })
+ // This error is raised inside the transport, after the SSE headers were
+ // staged. The route's four earlier rejections run before that staging and
+ // already answer JSON, so only this one proves the repair.
+ expect(res.status).toBe(400)
+ expect(res.headers['content-type']).toMatch(/^application\/json/)
+ expect(res.headers['cache-control']).toBeUndefined()
+ expect(res.body).toEqual({ error: 'invalid_request' })
+ // Witness: the attempt line is still emitted for the rejected request.
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ providerAttemptId: 'att-invalid',
+ outcome: 'failed',
+ code: 'invalid_request',
+ deliveredAs: 'http_status',
+ httpStatus: 400,
+ })
+ // The parser's message names caller-supplied fields, so it is withheld.
+ expect('reason' in lines[0]!).toBe(false)
+ expect(JSON.stringify(lines[0])).not.toContain(secretField)
+ expectNoForbiddenKeys(lines[0]!)
+ })
+
+ it('(d) labels the failure metric with a known transport code', async () => {
+ const { res, lines, metricsText } = await run({
+ providerAttemptId: 'att-buffer',
+ fetchFn: unterminatedUpstream(),
+ })
+ expect(res.status).toBe(503)
+ expect(res.body).toEqual({ error: 'sse_buffer_exceeded' })
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({
+ outcome: 'failed',
+ code: 'sse_buffer_exceeded',
+ httpStatus: 503,
+ })
+ expect(failureCount(metricsText, 'sse_buffer_exceeded')).toBe(1)
+ expect(failureCount(metricsText, 'other')).toBe(0)
+ })
+
+ it('(e) labels an unknown control-api code as other and keeps the raw code in the log', async () => {
+ const rawCode = 'k8s says: pod foo not found'
+ const { res, lines, metricsText } = await run({
+ providerAttemptId: 'att-unknown-code',
+ deniedCode: rawCode,
+ })
+ expect(res.status).toBe(503)
+ // Witness: the attempt failed and was logged with the code control-api sent.
+ expect(lines).toHaveLength(1)
+ expect(lines[0]).toMatchObject({ outcome: 'failed', code: rawCode, httpStatus: 503 })
+ expect(failureCount(metricsText, 'other')).toBe(1)
+ expect(metricsText).not.toContain(rawCode)
+ })
+})