Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(grok-llm-proxy): count tool-call arguments in UTF-8 bytes (#739)
Alfredo Lopeza76e767807d7
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex d650778ef..bc0003c6e 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -264,9 +264,9 @@ Stable codes: `insufficient_scope`, `no_grant`, `model_not_allowed`,not failover-eligible (failover class `null`), so the task fails with thatcode instead of `LLM_MODEL_OVERLOADED`.- `tool_call_arguments_exceeded`: the `arguments` text retained across one- response's pending tool calls crossed `MAX_TOOL_CALL_ARGUMENT_CHARS`+ response's pending tool calls crossed `MAX_TOOL_CALL_ARGUMENT_BYTES`(`grok-llm-proxy/src/grokTransport.ts`, equal to `maxRequestBodyBytes`,- 8 MiB). `maxToolCalls` bounds how+ 8 MiB, counted in UTF-8 bytes). `maxToolCalls` bounds howmany calls a response may carry, never how large each one is, and the SSEbuffer guard cannot see this: it bounds the unparsed tail between two `\n\n`boundaries and is reset on every read. Delivered likediff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.tsindex 217e340ad..691588752 100644--- a/grok-llm-proxy/src/grokTransport.ts+++ b/grok-llm-proxy/src/grokTransport.ts@@ -50,7 +50,7 @@ export const GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED: boolean = false/*** Ceiling on the combined `arguments` text retained for every pending tool call- * of one response, in UTF-16 code units.+ * of one response, in UTF-8 bytes.** `LIMITS.maxToolCalls` bounds how many calls a response may carry, never how* large each one is, and the SSE buffer guard cannot cover this: it bounds the@@ -58,12 +58,12 @@ export const GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED: boolean = false* so a long run of `response.function_call_arguments.delta` events grows the* proxy's heap and the response body without any ceiling.*- * Read from `LIMITS.maxRequestBodyBytes` so a response cannot be larger than a- * request the Host is allowed to send back (#731). The retained text is held- * in memory until the response ends, so raising the contract cap raises this- * per-stream heap ceiling with it.+ * Read from `LIMITS.maxRequestBodyBytes` (#731) and counted in the same unit,+ * UTF-8 bytes (R9-5). The retained text is held in memory until the response+ * ends, so raising the contract cap raises this per-stream heap ceiling with+ * it.*/-export const MAX_TOOL_CALL_ARGUMENT_CHARS: number = LIMITS.maxRequestBodyBytes+export const MAX_TOOL_CALL_ARGUMENT_BYTES: number = LIMITS.maxRequestBodyBytesexport class GrokTransportError extends Error {constructor(@@ -397,7 +397,7 @@ async function consumeSse(const reader = body.getReader()const decoder = new TextDecoder()const pending = new Map<string, PendingToolCall>()- const argumentBudget: ToolArgumentBudget = { chars: 0 }+ const argumentBudget: ToolArgumentBudget = { bytes: 0 }// Text stays streaming. Calls become executable only once the entire// response succeeds and its independent call budget has been validated.const toolFrames: Array<Extract<StreamFrame, { type: 'tool_call' }>> = []@@ -537,11 +537,13 @@ type PendingToolCall = {id: stringname: stringarguments: string+ /** UTF-8 bytes of `arguments`, kept so a delta costs only its own length. */+ argumentBytes: numberemitted: boolean}-/** Running total of the `arguments` text retained across `pending`. */-type ToolArgumentBudget = { chars: number }+/** Running total, in UTF-8 bytes, of the `arguments` text retained across `pending`. */+type ToolArgumentBudget = { bytes: number }function mapUpstreamEvent(event: unknown,@@ -614,29 +616,33 @@ function upsertPendingTool(id: String(source.call_id || source.id || key),name: 'tool',arguments: '',+ argumentBytes: 0,emitted: false,} satisfies PendingToolCall)if (typeof source.call_id === 'string' && source.call_id.trim()) current.id = source.call_idif (typeof source.name === 'string' && source.name.trim()) current.name = source.nameconst rawArgs = source.arguments- const retainedBefore = current.arguments.length+ const retainedBefore = current.argumentBytesif (typeof rawArgs === 'string') {// A closing event with empty or whitespace-only arguments carries nothing// new; it must not replace the buffer the deltas built, truncated or not.- current.arguments = source.append- ? `${current.arguments}${rawArgs}`- : rawArgs.trim()- ? rawArgs- : current.arguments+ if (source.append) {+ current.arguments = `${current.arguments}${rawArgs}`+ current.argumentBytes += Buffer.byteLength(rawArgs, 'utf8')+ } else if (rawArgs.trim()) {+ current.arguments = rawArgs+ current.argumentBytes = Buffer.byteLength(rawArgs, 'utf8')+ }} else if (isPlainObject(rawArgs) && !source.append) {current.arguments = JSON.stringify(rawArgs)+ current.argumentBytes = Buffer.byteLength(current.arguments, 'utf8')}- budget.chars += current.arguments.length - retainedBefore- if (budget.chars > MAX_TOOL_CALL_ARGUMENT_CHARS) {+ budget.bytes += current.argumentBytes - retainedBefore+ if (budget.bytes > MAX_TOOL_CALL_ARGUMENT_BYTES) {throw new GrokTransportError('tool_call_arguments_exceeded',- `tool call arguments exceed ${MAX_TOOL_CALL_ARGUMENT_CHARS} characters`,- { limit: MAX_TOOL_CALL_ARGUMENT_CHARS, observed: budget.chars }+ `tool call arguments exceed ${MAX_TOOL_CALL_ARGUMENT_BYTES} bytes`,+ { limit: MAX_TOOL_CALL_ARGUMENT_BYTES, observed: budget.bytes })}pending.set(key, current)diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.tsindex 9d2075e73..07ff7619e 100644--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts@@ -8,7 +8,7 @@ import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'import {GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED,GrokTransportError,- MAX_TOOL_CALL_ARGUMENT_CHARS,+ MAX_TOOL_CALL_ARGUMENT_BYTES,type StreamFrame,type StreamGrokCompletionInput,streamGrokCompletion,@@ -95,12 +95,12 @@ const ARGUMENT_CHUNK_CHARS = 65_536/*** `{"q":"aaa…"}` split into `ARGUMENT_CHUNK_CHARS`-sized deltas, sized so the- * retained total lands exactly on `MAX_TOOL_CALL_ARGUMENT_CHARS`. Valid JSON,+ * retained total lands exactly on `MAX_TOOL_CALL_ARGUMENT_BYTES`. Valid JSON,* so the at-limit case can assert the parsed arguments the sink receives.*/function argumentDeltas(): string[] {const envelopeChars = '{"q":""}'.length- const full = `{"q":"${'a'.repeat(MAX_TOOL_CALL_ARGUMENT_CHARS - envelopeChars)}"}`+ const full = `{"q":"${'a'.repeat(MAX_TOOL_CALL_ARGUMENT_BYTES - envelopeChars)}"}`const chunks: string[] = []for (let offset = 0; offset < full.length; offset += ARGUMENT_CHUNK_CHARS) {chunks.push(full.slice(offset, offset + ARGUMENT_CHUNK_CHARS))@@ -120,12 +120,12 @@ function argumentFrames(deltas: string[]): string[] {}describe('streamGrokCompletion', () => {- // R3-1 (#731): a response may carry tool-call arguments as large as the- // request the Host is allowed to send back, and no larger. Held equal to the+ // R3-1 (#731): the retained tool-call arguments of one response are bounded+ // by the contract request cap, in UTF-8 bytes (R9-5). Held equal to the// contract so raising the request cap moves this bound with it.it('T-R3-1d bounds tool-call arguments at the contract request cap', () => {- expect(MAX_TOOL_CALL_ARGUMENT_CHARS).toBe(LIMITS.maxRequestBodyBytes)- expect(MAX_TOOL_CALL_ARGUMENT_CHARS).toBe(8 * 1024 * 1024)+ expect(MAX_TOOL_CALL_ARGUMENT_BYTES).toBe(LIMITS.maxRequestBodyBytes)+ expect(MAX_TOOL_CALL_ARGUMENT_BYTES).toBe(8 * 1024 * 1024)})it('does not read further upstream bytes while the frame consumer is back-pressured', async () => {@@ -476,7 +476,7 @@ describe('streamGrokCompletion', () => {// guard cannot see it: it is reset on every `\n\n` boundary.it('accepts tool-call arguments that land exactly on the retained budget', async () => {const deltas = argumentDeltas()- expect(deltas.join('')).toHaveLength(MAX_TOOL_CALL_ARGUMENT_CHARS)+ expect(deltas.join('')).toHaveLength(MAX_TOOL_CALL_ARGUMENT_BYTES)const emitted: StreamFrame[] = []const frames = [`data: ${JSON.stringify({ type: 'response.output_text.delta', delta: 'before' })}\n\n`,@@ -514,7 +514,7 @@ describe('streamGrokCompletion', () => {const toolCalls = emitted.flatMap(frame => (frame.type === 'tool_call' ? [frame] : []))expect(toolCalls).toHaveLength(1)expect(toolCalls[0]?.arguments).toEqual({- q: 'a'.repeat(MAX_TOOL_CALL_ARGUMENT_CHARS - '{"q":""}'.length),+ q: 'a'.repeat(MAX_TOOL_CALL_ARGUMENT_BYTES - '{"q":""}'.length),})})@@ -558,8 +558,8 @@ describe('streamGrokCompletion', () => {await expect(pending).rejects.toMatchObject({code: 'tool_call_arguments_exceeded',details: {- limit: MAX_TOOL_CALL_ARGUMENT_CHARS,- observed: MAX_TOOL_CALL_ARGUMENT_CHARS + ARGUMENT_CHUNK_CHARS,+ limit: MAX_TOOL_CALL_ARGUMENT_BYTES,+ observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_CHUNK_CHARS,},})expect(emitted.filter(frame => frame.type === 'tool_call')).toHaveLength(0)@@ -580,9 +580,9 @@ describe('streamGrokCompletion', () => {// half the budget in code units and two bytes over it in UTF-8.it('T-R9-5 counts retained tool-call arguments in UTF-8 bytes', async () => {const envelope = '{"q":""}'- const full = `{"q":"${'é'.repeat((MAX_TOOL_CALL_ARGUMENT_CHARS - envelope.length) / 2 + 1)}"}`- expect(full.length).toBeLessThan(MAX_TOOL_CALL_ARGUMENT_CHARS)- expect(Buffer.byteLength(full, 'utf8')).toBe(MAX_TOOL_CALL_ARGUMENT_CHARS + 2)+ const full = `{"q":"${'é'.repeat((MAX_TOOL_CALL_ARGUMENT_BYTES - envelope.length) / 2 + 1)}"}`+ expect(full.length).toBeLessThan(MAX_TOOL_CALL_ARGUMENT_BYTES)+ expect(Buffer.byteLength(full, 'utf8')).toBe(MAX_TOOL_CALL_ARGUMENT_BYTES + 2)const deltas: string[] = []for (let offset = 0; offset < full.length; offset += ARGUMENT_CHUNK_CHARS) {deltas.push(full.slice(offset, offset + ARGUMENT_CHUNK_CHARS))@@ -624,7 +624,7 @@ describe('streamGrokCompletion', () => {})await expect(pending).rejects.toMatchObject({code: 'tool_call_arguments_exceeded',- details: { limit: MAX_TOOL_CALL_ARGUMENT_CHARS, observed: MAX_TOOL_CALL_ARGUMENT_CHARS + 2 },+ details: { limit: MAX_TOOL_CALL_ARGUMENT_BYTES, observed: MAX_TOOL_CALL_ARGUMENT_BYTES + 2 },})expect(emitted.filter(frame => frame.type === 'tool_call')).toHaveLength(0)// Liveness witness: the leading text proves the stream was read.diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.tsindex 50e37bc0b..ed58c64b8 100644--- a/grok-llm-proxy/test/server.security.test.ts+++ b/grok-llm-proxy/test/server.security.test.ts@@ -16,7 +16,7 @@ import {type FinalizeAttemptSuccess,type RedeemAttemptSuccess,} from '../src/controlApiClient.js'-import { MAX_TOOL_CALL_ARGUMENT_CHARS } from '../src/grokTransport.js'+import { MAX_TOOL_CALL_ARGUMENT_BYTES } from '../src/grokTransport.js'import { REDACT_PATHS, logger } from '../src/logger.js'import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'import { ENVELOPE_ALLOWANCE_BYTES } from '../src/requestLimits.js'@@ -618,7 +618,7 @@ describe('grok-llm-proxy attempt telemetry', () => {const ARGUMENT_OVERRUN_CHUNK = 65_536function oversizedArgumentsUpstream(): typeof fetch {- const chunks = MAX_TOOL_CALL_ARGUMENT_CHARS / ARGUMENT_OVERRUN_CHUNK + 1+ const chunks = MAX_TOOL_CALL_ARGUMENT_BYTES / ARGUMENT_OVERRUN_CHUNK + 1const frames = [`data: ${JSON.stringify({type: 'response.output_item.added',@@ -793,8 +793,8 @@ describe('grok-llm-proxy attempt telemetry', () => {outcome: 'failed',code: 'tool_call_arguments_exceeded',details: {- limit: MAX_TOOL_CALL_ARGUMENT_CHARS,- observed: MAX_TOOL_CALL_ARGUMENT_CHARS + ARGUMENT_OVERRUN_CHUNK,+ limit: MAX_TOOL_CALL_ARGUMENT_BYTES,+ observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_OVERRUN_CHUNK,},deliveredAs: 'http_status',httpStatus: 422,