Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(grok): cover the attempt line, the failure metric and the error body
Alfredo Lopez4e1a5a587b85
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex 6e53b9461..3771605df 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from 'vitest'+import { describe, expect, it, vi } from 'vitest'import jwt from 'jsonwebtoken'import { generateKeyPairSync } from 'node:crypto'import request from 'supertest'@@ -9,7 +9,14 @@ import {import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'import { type GrokLlmProxyConfig, loadConfig } from '../src/config.js'-import { ControlApiClient, ControlApiClientError } from '../src/controlApiClient.js'+import {+ ControlApiClient,+ ControlApiClientError,+ type FinalizeAttemptSuccess,+ type RedeemAttemptSuccess,+} from '../src/controlApiClient.js'+import { logger } from '../src/logger.js'+import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'import { createProxyApps } from '../src/server.js'const { privateKey, publicKey } = generateKeyPairSync('rsa', {@@ -445,3 +452,290 @@ describe('grok-llm-proxy startup config', () => {})})++describe('grok-llm-proxy attempt telemetry', () => {+ const lookup = async () => [{ address: '1.2.3.4', family: 4 }]+ // Keys that carry caller content. The attempt line is identifiers and counts;+ // none of these may ever reach it.+ const FORBIDDEN_LOG_KEYS = ['body', 'request', 'executionTicket', 'arguments']++ function completionBody(+ providerAttemptId: string,+ tamper?: (raw: Record<string, unknown>) => void+ ): Record<string, unknown> {+ const raw: Record<string, unknown> = {+ schemaVersion: 'grok-completion-request.v1',+ requestId: `req-${providerAttemptId}`,+ idempotencyKey: `idem-${providerAttemptId}`,+ provider: 'grok-subscription',+ model: 'gpt-5.1',+ messages: [{ role: 'user', content: 'hello' }],+ }+ const parsed = parseGrokCompletionRequestV1(raw)+ if (!parsed.ok) throw new Error(parsed.message)+ const requestHash = hashGrokCompletionRequestV1(parsed.value)+ // Applied after hashing: the ticket stays bound to the untampered request,+ // so the transport's parser rejects the body before any hash comparison.+ tamper?.(raw)+ const executionTicket = sign(+ {+ jti: '55555555-5555-4555-8555-555555555555',+ typ: 'grok-execution-ticket',+ hostRef: 'research-host',+ model: 'gpt-5.1',+ requestHash,+ providerAttemptId,+ },+ 'grok-llm-proxy'+ )+ return { executionTicket, requestHash, request: raw }+ }++ function grantingClient(): { client: ControlApiClient; receipts: unknown[] } {+ const receipts: unknown[] = []+ const client = {+ async redeem(): Promise<RedeemAttemptSuccess> {+ return {+ accessToken: 'test-access-telemetry',+ transport: {+ protocolVersion: 'grok-subscription-transport.v1',+ completionsOrigin: GROK_COMPLETIONS_ORIGIN,+ catalogOrigin: GROK_CATALOG_ORIGIN,+ operation: 'completion_stream',+ servedModel: 'gpt-5.1',+ maxStreamDurationMs: 300_000,+ },+ expiryClass: 'short_lived',+ attemptReceipt: 'c'.repeat(64),+ }+ },+ async finalize(input: {+ receipt: { outcome: FinalizeAttemptSuccess['outcome'] }+ }): Promise<FinalizeAttemptSuccess> {+ receipts.push(input.receipt)+ return { providerAttemptId: 'att', outcome: input.receipt.outcome, duplicate: false }+ },+ } as unknown as ControlApiClient+ return { client, receipts }+ }++ function upstream(textDeltas: number, calls: number): typeof fetch {+ const frames: string[] = []+ for (let index = 0; index < textDeltas; index += 1) {+ frames.push(+ `data: ${JSON.stringify({ type: 'response.output_text.delta', delta: `t${index}` })}\n\n`+ )+ }+ for (let index = 0; index < calls; index += 1) {+ frames.push(+ `data: ${JSON.stringify({+ type: 'response.output_item.done',+ item: { type: 'function_call', id: `call-${index}`, name: 'lookup', arguments: '{}' },+ })}\n\n`+ )+ }+ frames.push('data: {"type":"response.completed"}\n\n')+ return (async () =>+ new Response(frames.join(''), {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })) as typeof fetch+ }++ // One SSE block larger than the transport's 1 MiB buffer and never terminated,+ // so the buffer guard fires before any frame is delivered.+ function unterminatedUpstream(): typeof fetch {+ return (async () =>+ new Response(`data: ${'x'.repeat(1_100_000)}`, {+ status: 200,+ headers: { 'content-type': 'text/event-stream' },+ })) as typeof fetch+ }++ function denyingClient(code: string): ControlApiClient {+ return {+ async redeem() {+ throw new ControlApiClientError(code, 'control API request denied')+ },+ async finalize() {+ throw new Error('finalize must not run after a denied redeem')+ },+ } as unknown as ControlApiClient+ }++ function failureCount(metricsText: string, code: string): number {+ const line = metricsText+ .split('\n')+ .find(row => row.startsWith(`grok_proxy_attempt_failures_total{code="${code}"}`))+ return line ? Number(line.split(' ').pop()) : 0+ }++ async function run(options: {+ providerAttemptId: string+ textDeltas?: number+ calls?: number+ tamper?: (raw: Record<string, unknown>) => void+ deniedCode?: string+ fetchFn?: typeof fetch+ }) {+ const info = vi.spyOn(logger, 'info')+ const warn = vi.spyOn(logger, 'warn')+ const { client, receipts } = grantingClient()+ const apps = createProxyApps(config({ maxBodyBytes: 65_536 }), {+ controlApiClient: options.deniedCode ? denyingClient(options.deniedCode) : client,+ fetchFn: options.fetchFn ?? upstream(options.textDeltas ?? 0, options.calls ?? 0),+ lookup,+ })+ try {+ const res = await request(apps.runtimeApp)+ .post('/internal/runtime/v1/grok/completions')+ .set('Authorization', `Bearer ${platformToken()}`)+ .send(completionBody(options.providerAttemptId, options.tamper))+ const metricsText = (await request(apps.probeApp).get('/metrics')).text+ const lines = [...info.mock.calls, ...warn.mock.calls]+ .map(call => call[0] as unknown as Record<string, unknown>)+ .filter(entry => entry?.event === 'grok_proxy_attempt_finished')+ return { res, receipts, lines, metricsText }+ } finally {+ info.mockRestore()+ warn.mockRestore()+ }+ }++ function expectNoForbiddenKeys(line: Record<string, unknown>): void {+ for (const key of FORBIDDEN_LOG_KEYS) expect(key in line).toBe(false)+ }++ it('(a) answers 422 and logs one attempt line when 65 calls arrive before any text', async () => {+ const { res, receipts, lines, metricsText } = await run({+ providerAttemptId: 'att-limit-http',+ calls: 65,+ })+ expect(res.status).toBe(422)+ // The staged SSE headers are replaced by a JSON response.+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.headers['cache-control']).toBeUndefined()+ expect(res.body).toEqual({ error: 'tool_call_limit_exceeded' })+ expect(receipts).toEqual([expect.objectContaining({ outcome: 'error' })])+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-limit-http',+ outcome: 'failed',+ code: 'tool_call_limit_exceeded',+ reason: 'tool calls exceed 64',+ details: { limit: 64, observed: 65 },+ deliveredAs: 'http_status',+ httpStatus: 422,+ toolCalls: 0,+ textChunks: 0,+ })+ expectNoForbiddenKeys(lines[0]!)+ expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(1)+ })++ it('(a2) sends an SSE error frame when text was already streamed', async () => {+ const { res, lines } = await run({+ providerAttemptId: 'att-limit-sse',+ textDeltas: 1,+ calls: 65,+ })+ expect(res.status).toBe(200)+ expect(res.text).toContain('data: {"type":"text","text":"t0"}')+ expect(res.text).toContain('data: {"type":"error","code":"tool_call_limit_exceeded"}')+ expect(res.text).not.toContain('"type":"tool_call"')+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ outcome: 'failed',+ code: 'tool_call_limit_exceeded',+ deliveredAs: 'sse_error',+ textChunks: 1,+ toolCalls: 0,+ })+ expect('httpStatus' in lines[0]!).toBe(false)+ expectNoForbiddenKeys(lines[0]!)+ })++ it('(b) logs the counted frames of a successful attempt', async () => {+ const { res, lines, metricsText } = await run({+ providerAttemptId: 'att-success',+ calls: 3,+ })+ expect(res.status).toBe(200)+ expect(res.text).toContain('"type":"done","outcome":"success"')+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-success',+ outcome: 'success',+ deliveredAs: 'sse_done',+ toolCalls: 3,+ textChunks: 0,+ })+ expectNoForbiddenKeys(lines[0]!)+ // Witness: the attempt was counted, so the absent failure sample is meaningful.+ expect(metricsText).toMatch(+ /grok_llm_proxy_attempts_total\{outcome="success",operation="completion_stream"\} 1/+ )+ expect(failureCount(metricsText, 'tool_call_limit_exceeded')).toBe(0)+ })++ it('(c) serves a post-staging invalid request as JSON without the parse message', async () => {+ const secretField = 'sk-live-0123456789abcdef'+ const { res, lines } = await run({+ providerAttemptId: 'att-invalid',+ tamper: raw => {+ ;(raw.messages as Record<string, unknown>[])[0]![secretField] = 'x'+ },+ })+ // This error is raised inside the transport, after the SSE headers were+ // staged. The route's four earlier rejections run before that staging and+ // already answer JSON, so only this one proves the repair.+ expect(res.status).toBe(400)+ expect(res.headers['content-type']).toMatch(/^application\/json/)+ expect(res.headers['cache-control']).toBeUndefined()+ expect(res.body).toEqual({ error: 'invalid_request' })+ // Witness: the attempt line is still emitted for the rejected request.+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ providerAttemptId: 'att-invalid',+ outcome: 'failed',+ code: 'invalid_request',+ deliveredAs: 'http_status',+ httpStatus: 400,+ })+ // The parser's message names caller-supplied fields, so it is withheld.+ expect('reason' in lines[0]!).toBe(false)+ expect(JSON.stringify(lines[0])).not.toContain(secretField)+ expectNoForbiddenKeys(lines[0]!)+ })++ it('(d) labels the failure metric with a known transport code', async () => {+ const { res, lines, metricsText } = await run({+ providerAttemptId: 'att-buffer',+ fetchFn: unterminatedUpstream(),+ })+ expect(res.status).toBe(503)+ expect(res.body).toEqual({ error: 'sse_buffer_exceeded' })+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({+ outcome: 'failed',+ code: 'sse_buffer_exceeded',+ httpStatus: 503,+ })+ expect(failureCount(metricsText, 'sse_buffer_exceeded')).toBe(1)+ expect(failureCount(metricsText, 'other')).toBe(0)+ })++ it('(e) labels an unknown control-api code as other and keeps the raw code in the log', async () => {+ const rawCode = 'k8s says: pod foo not found'+ const { res, lines, metricsText } = await run({+ providerAttemptId: 'att-unknown-code',+ deniedCode: rawCode,+ })+ expect(res.status).toBe(503)+ // Witness: the attempt failed and was logged with the code control-api sent.+ expect(lines).toHaveLength(1)+ expect(lines[0]).toMatchObject({ outcome: 'failed', code: rawCode, httpStatus: 503 })+ expect(failureCount(metricsText, 'other')).toBe(1)+ expect(metricsText).not.toContain(rawCode)+ })+})