Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(grok-llm-proxy): count tool-call arguments in UTF-8 bytes (#739)

Alfredo Lopeza76e767807d7
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index d650778ef..bc0003c6e 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -264,9 +264,9 @@ Stable codes: `insufficient_scope`, `no_grant`, `model_not_allowed`,
not failover-eligible (failover class `null`), so the task fails with that
code instead of `LLM_MODEL_OVERLOADED`.
- `tool_call_arguments_exceeded`: the `arguments` text retained across one
- response's pending tool calls crossed `MAX_TOOL_CALL_ARGUMENT_CHARS`
+ response's pending tool calls crossed `MAX_TOOL_CALL_ARGUMENT_BYTES`
(`grok-llm-proxy/src/grokTransport.ts`, equal to `maxRequestBodyBytes`,
- 8 MiB). `maxToolCalls` bounds how
+ 8 MiB, counted in UTF-8 bytes). `maxToolCalls` bounds how
many calls a response may carry, never how large each one is, and the SSE
buffer guard cannot see this: it bounds the unparsed tail between two `\n\n`
boundaries and is reset on every read. Delivered like
diff --git a/grok-llm-proxy/src/grokTransport.ts b/grok-llm-proxy/src/grokTransport.ts
index 217e340ad..691588752 100644
--- a/grok-llm-proxy/src/grokTransport.ts
+++ b/grok-llm-proxy/src/grokTransport.ts
@@ -50,7 +50,7 @@ export const GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED: boolean = false
/**
* Ceiling on the combined `arguments` text retained for every pending tool call
- * of one response, in UTF-16 code units.
+ * of one response, in UTF-8 bytes.
*
* `LIMITS.maxToolCalls` bounds how many calls a response may carry, never how
* large each one is, and the SSE buffer guard cannot cover this: it bounds the
@@ -58,12 +58,12 @@ export const GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED: boolean = false
* so a long run of `response.function_call_arguments.delta` events grows the
* proxy's heap and the response body without any ceiling.
*
- * Read from `LIMITS.maxRequestBodyBytes` so a response cannot be larger than a
- * request the Host is allowed to send back (#731). The retained text is held
- * in memory until the response ends, so raising the contract cap raises this
- * per-stream heap ceiling with it.
+ * Read from `LIMITS.maxRequestBodyBytes` (#731) and counted in the same unit,
+ * UTF-8 bytes (R9-5). The retained text is held in memory until the response
+ * ends, so raising the contract cap raises this per-stream heap ceiling with
+ * it.
*/
-export const MAX_TOOL_CALL_ARGUMENT_CHARS: number = LIMITS.maxRequestBodyBytes
+export const MAX_TOOL_CALL_ARGUMENT_BYTES: number = LIMITS.maxRequestBodyBytes
export class GrokTransportError extends Error {
constructor(
@@ -397,7 +397,7 @@ async function consumeSse(
const reader = body.getReader()
const decoder = new TextDecoder()
const pending = new Map<string, PendingToolCall>()
- const argumentBudget: ToolArgumentBudget = { chars: 0 }
+ const argumentBudget: ToolArgumentBudget = { bytes: 0 }
// Text stays streaming. Calls become executable only once the entire
// response succeeds and its independent call budget has been validated.
const toolFrames: Array<Extract<StreamFrame, { type: 'tool_call' }>> = []
@@ -537,11 +537,13 @@ type PendingToolCall = {
id: string
name: string
arguments: string
+ /** UTF-8 bytes of `arguments`, kept so a delta costs only its own length. */
+ argumentBytes: number
emitted: boolean
}
-/** Running total of the `arguments` text retained across `pending`. */
-type ToolArgumentBudget = { chars: number }
+/** Running total, in UTF-8 bytes, of the `arguments` text retained across `pending`. */
+type ToolArgumentBudget = { bytes: number }
function mapUpstreamEvent(
event: unknown,
@@ -614,29 +616,33 @@ function upsertPendingTool(
id: String(source.call_id || source.id || key),
name: 'tool',
arguments: '',
+ argumentBytes: 0,
emitted: false,
} satisfies PendingToolCall)
if (typeof source.call_id === 'string' && source.call_id.trim()) current.id = source.call_id
if (typeof source.name === 'string' && source.name.trim()) current.name = source.name
const rawArgs = source.arguments
- const retainedBefore = current.arguments.length
+ const retainedBefore = current.argumentBytes
if (typeof rawArgs === 'string') {
// A closing event with empty or whitespace-only arguments carries nothing
// new; it must not replace the buffer the deltas built, truncated or not.
- current.arguments = source.append
- ? `${current.arguments}${rawArgs}`
- : rawArgs.trim()
- ? rawArgs
- : current.arguments
+ if (source.append) {
+ current.arguments = `${current.arguments}${rawArgs}`
+ current.argumentBytes += Buffer.byteLength(rawArgs, 'utf8')
+ } else if (rawArgs.trim()) {
+ current.arguments = rawArgs
+ current.argumentBytes = Buffer.byteLength(rawArgs, 'utf8')
+ }
} else if (isPlainObject(rawArgs) && !source.append) {
current.arguments = JSON.stringify(rawArgs)
+ current.argumentBytes = Buffer.byteLength(current.arguments, 'utf8')
}
- budget.chars += current.arguments.length - retainedBefore
- if (budget.chars > MAX_TOOL_CALL_ARGUMENT_CHARS) {
+ budget.bytes += current.argumentBytes - retainedBefore
+ if (budget.bytes > MAX_TOOL_CALL_ARGUMENT_BYTES) {
throw new GrokTransportError(
'tool_call_arguments_exceeded',
- `tool call arguments exceed ${MAX_TOOL_CALL_ARGUMENT_CHARS} characters`,
- { limit: MAX_TOOL_CALL_ARGUMENT_CHARS, observed: budget.chars }
+ `tool call arguments exceed ${MAX_TOOL_CALL_ARGUMENT_BYTES} bytes`,
+ { limit: MAX_TOOL_CALL_ARGUMENT_BYTES, observed: budget.bytes }
)
}
pending.set(key, current)
diff --git a/grok-llm-proxy/test/grokTransport.conformance.test.ts b/grok-llm-proxy/test/grokTransport.conformance.test.ts
index 9d2075e73..07ff7619e 100644
--- a/grok-llm-proxy/test/grokTransport.conformance.test.ts
+++ b/grok-llm-proxy/test/grokTransport.conformance.test.ts
@@ -8,7 +8,7 @@ import type { RedeemAttemptSuccess } from '../src/controlApiClient.js'
import {
GROK_UPSTREAM_TEMPERATURE_PROBE_CONFIRMED,
GrokTransportError,
- MAX_TOOL_CALL_ARGUMENT_CHARS,
+ MAX_TOOL_CALL_ARGUMENT_BYTES,
type StreamFrame,
type StreamGrokCompletionInput,
streamGrokCompletion,
@@ -95,12 +95,12 @@ const ARGUMENT_CHUNK_CHARS = 65_536
/**
* `{"q":"aaa…"}` split into `ARGUMENT_CHUNK_CHARS`-sized deltas, sized so the
- * retained total lands exactly on `MAX_TOOL_CALL_ARGUMENT_CHARS`. Valid JSON,
+ * retained total lands exactly on `MAX_TOOL_CALL_ARGUMENT_BYTES`. Valid JSON,
* so the at-limit case can assert the parsed arguments the sink receives.
*/
function argumentDeltas(): string[] {
const envelopeChars = '{"q":""}'.length
- const full = `{"q":"${'a'.repeat(MAX_TOOL_CALL_ARGUMENT_CHARS - envelopeChars)}"}`
+ const full = `{"q":"${'a'.repeat(MAX_TOOL_CALL_ARGUMENT_BYTES - envelopeChars)}"}`
const chunks: string[] = []
for (let offset = 0; offset < full.length; offset += ARGUMENT_CHUNK_CHARS) {
chunks.push(full.slice(offset, offset + ARGUMENT_CHUNK_CHARS))
@@ -120,12 +120,12 @@ function argumentFrames(deltas: string[]): string[] {
}
describe('streamGrokCompletion', () => {
- // R3-1 (#731): a response may carry tool-call arguments as large as the
- // request the Host is allowed to send back, and no larger. Held equal to the
+ // R3-1 (#731): the retained tool-call arguments of one response are bounded
+ // by the contract request cap, in UTF-8 bytes (R9-5). Held equal to the
// contract so raising the request cap moves this bound with it.
it('T-R3-1d bounds tool-call arguments at the contract request cap', () => {
- expect(MAX_TOOL_CALL_ARGUMENT_CHARS).toBe(LIMITS.maxRequestBodyBytes)
- expect(MAX_TOOL_CALL_ARGUMENT_CHARS).toBe(8 * 1024 * 1024)
+ expect(MAX_TOOL_CALL_ARGUMENT_BYTES).toBe(LIMITS.maxRequestBodyBytes)
+ expect(MAX_TOOL_CALL_ARGUMENT_BYTES).toBe(8 * 1024 * 1024)
})
it('does not read further upstream bytes while the frame consumer is back-pressured', async () => {
@@ -476,7 +476,7 @@ describe('streamGrokCompletion', () => {
// guard cannot see it: it is reset on every `\n\n` boundary.
it('accepts tool-call arguments that land exactly on the retained budget', async () => {
const deltas = argumentDeltas()
- expect(deltas.join('')).toHaveLength(MAX_TOOL_CALL_ARGUMENT_CHARS)
+ expect(deltas.join('')).toHaveLength(MAX_TOOL_CALL_ARGUMENT_BYTES)
const emitted: StreamFrame[] = []
const frames = [
`data: ${JSON.stringify({ type: 'response.output_text.delta', delta: 'before' })}\n\n`,
@@ -514,7 +514,7 @@ describe('streamGrokCompletion', () => {
const toolCalls = emitted.flatMap(frame => (frame.type === 'tool_call' ? [frame] : []))
expect(toolCalls).toHaveLength(1)
expect(toolCalls[0]?.arguments).toEqual({
- q: 'a'.repeat(MAX_TOOL_CALL_ARGUMENT_CHARS - '{"q":""}'.length),
+ q: 'a'.repeat(MAX_TOOL_CALL_ARGUMENT_BYTES - '{"q":""}'.length),
})
})
@@ -558,8 +558,8 @@ describe('streamGrokCompletion', () => {
await expect(pending).rejects.toMatchObject({
code: 'tool_call_arguments_exceeded',
details: {
- limit: MAX_TOOL_CALL_ARGUMENT_CHARS,
- observed: MAX_TOOL_CALL_ARGUMENT_CHARS + ARGUMENT_CHUNK_CHARS,
+ limit: MAX_TOOL_CALL_ARGUMENT_BYTES,
+ observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_CHUNK_CHARS,
},
})
expect(emitted.filter(frame => frame.type === 'tool_call')).toHaveLength(0)
@@ -580,9 +580,9 @@ describe('streamGrokCompletion', () => {
// half the budget in code units and two bytes over it in UTF-8.
it('T-R9-5 counts retained tool-call arguments in UTF-8 bytes', async () => {
const envelope = '{"q":""}'
- const full = `{"q":"${'é'.repeat((MAX_TOOL_CALL_ARGUMENT_CHARS - envelope.length) / 2 + 1)}"}`
- expect(full.length).toBeLessThan(MAX_TOOL_CALL_ARGUMENT_CHARS)
- expect(Buffer.byteLength(full, 'utf8')).toBe(MAX_TOOL_CALL_ARGUMENT_CHARS + 2)
+ const full = `{"q":"${'é'.repeat((MAX_TOOL_CALL_ARGUMENT_BYTES - envelope.length) / 2 + 1)}"}`
+ expect(full.length).toBeLessThan(MAX_TOOL_CALL_ARGUMENT_BYTES)
+ expect(Buffer.byteLength(full, 'utf8')).toBe(MAX_TOOL_CALL_ARGUMENT_BYTES + 2)
const deltas: string[] = []
for (let offset = 0; offset < full.length; offset += ARGUMENT_CHUNK_CHARS) {
deltas.push(full.slice(offset, offset + ARGUMENT_CHUNK_CHARS))
@@ -624,7 +624,7 @@ describe('streamGrokCompletion', () => {
})
await expect(pending).rejects.toMatchObject({
code: 'tool_call_arguments_exceeded',
- details: { limit: MAX_TOOL_CALL_ARGUMENT_CHARS, observed: MAX_TOOL_CALL_ARGUMENT_CHARS + 2 },
+ details: { limit: MAX_TOOL_CALL_ARGUMENT_BYTES, observed: MAX_TOOL_CALL_ARGUMENT_BYTES + 2 },
})
expect(emitted.filter(frame => frame.type === 'tool_call')).toHaveLength(0)
// Liveness witness: the leading text proves the stream was read.
diff --git a/grok-llm-proxy/test/server.security.test.ts b/grok-llm-proxy/test/server.security.test.ts
index 50e37bc0b..ed58c64b8 100644
--- a/grok-llm-proxy/test/server.security.test.ts
+++ b/grok-llm-proxy/test/server.security.test.ts
@@ -16,7 +16,7 @@ import {
type FinalizeAttemptSuccess,
type RedeemAttemptSuccess,
} from '../src/controlApiClient.js'
-import { MAX_TOOL_CALL_ARGUMENT_CHARS } from '../src/grokTransport.js'
+import { MAX_TOOL_CALL_ARGUMENT_BYTES } from '../src/grokTransport.js'
import { REDACT_PATHS, logger } from '../src/logger.js'
import { GROK_CATALOG_ORIGIN, GROK_COMPLETIONS_ORIGIN } from '../src/originPolicy.js'
import { ENVELOPE_ALLOWANCE_BYTES } from '../src/requestLimits.js'
@@ -618,7 +618,7 @@ describe('grok-llm-proxy attempt telemetry', () => {
const ARGUMENT_OVERRUN_CHUNK = 65_536
function oversizedArgumentsUpstream(): typeof fetch {
- const chunks = MAX_TOOL_CALL_ARGUMENT_CHARS / ARGUMENT_OVERRUN_CHUNK + 1
+ const chunks = MAX_TOOL_CALL_ARGUMENT_BYTES / ARGUMENT_OVERRUN_CHUNK + 1
const frames = [
`data: ${JSON.stringify({
type: 'response.output_item.added',
@@ -793,8 +793,8 @@ describe('grok-llm-proxy attempt telemetry', () => {
outcome: 'failed',
code: 'tool_call_arguments_exceeded',
details: {
- limit: MAX_TOOL_CALL_ARGUMENT_CHARS,
- observed: MAX_TOOL_CALL_ARGUMENT_CHARS + ARGUMENT_OVERRUN_CHUNK,
+ limit: MAX_TOOL_CALL_ARGUMENT_BYTES,
+ observed: MAX_TOOL_CALL_ARGUMENT_BYTES + ARGUMENT_OVERRUN_CHUNK,
},
deliveredAs: 'http_status',
httpStatus: 422,