Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(mcp-host): pin the heuristic to the escaped UTF-8 bytes the contract measures (#731)
Alfredo Lopez1bcf5097ac6b
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 7289ca3b6..cd76b8dcf 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.380",+ "version": "0.1.381","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.380",+ "version": "0.1.381","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 107f771cd..81a9f76d3 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.380",+ "version": "0.1.381","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.tsindex a4500b063..ba0c43e0a 100644--- a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts+++ b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts@@ -59,6 +59,44 @@ describe('heuristicCount', () => {// the arguments: ~4,000 chars → ~1,000 tokens.expect(heuristicCount([msg])).toBeGreaterThanOrEqual(1_000)})++ // The provider attempt contract measures `Buffer.byteLength(JSON.stringify(request))`:+ // UTF-8 bytes of the JSON-escaped text. A count of UTF-16 code units under-reads+ // everything that is not plain ASCII, so the gauge sat below 0.8 while the+ // request was already over the cap (review r2, M2).+ const wireBytes = (s: string): number => Buffer.byteLength(JSON.stringify(s), 'utf8') - 2++ it('T-R2-1a counts CJK content by its UTF-8 bytes', () => {+ const content = '漢'.repeat(1_000) // 1,000 code units, 3,000 UTF-8 bytes+ expect(wireBytes(content)).toBe(3_000)+ expect(heuristicCount([{ role: 'tool', content, tool_call_id: 'c' }])).toBe(750 + 4)+ })++ it('T-R2-1b counts control characters by their JSON escapes', () => {+ const content = '\u0001'.repeat(1_000) // each serializes as `\u0001`, 6 bytes+ expect(wireBytes(content)).toBe(6_000)+ expect(heuristicCount([{ role: 'tool', content, tool_call_id: 'c' }])).toBe(1_500 + 4)+ })++ it('T-R2-1c counts the escaped quotes of minified JSON carried as a string', () => {+ const content = minifiedMcpResult(2, 8_000)+ // Witness that the payload has escapes at all, so the assertion below is not+ // satisfied by a plain length count.+ expect(wireBytes(content)).toBeGreaterThan(content.length)+ expect(heuristicCount([{ role: 'tool', content, tool_call_id: 'c' }])).toBe(+ Math.ceil(wireBytes(content) / 4) + 4+ )+ })++ it('T-R2-1d counts tool_calls arguments by their UTF-8 bytes', () => {+ const args = { q: '漢'.repeat(1_000) }+ const bytes = Buffer.byteLength(JSON.stringify(args), 'utf8') // 3,008+ expect(+ heuristicCount([+ { role: 'assistant', content: '', tool_calls: [{ id: 'x', name: 'y', arguments: args }] },+ ])+ ).toBe(4 + Math.ceil(bytes / 4))+ })})describe('heuristicCountTools', () => {@@ -96,4 +134,10 @@ describe('heuristicCountTools', () => {]expect(heuristicCountTools(large)).toBeGreaterThan(heuristicCountTools(small))})++ it('T-R2-1e counts a non-ASCII description by its UTF-8 bytes', () => {+ // name 1 + '\n' 1 + description 3,000 bytes + '\n' 1 + '{}' 2 = 3,005 → ceil/4 = 752, +4+ const tools = [{ name: 'a', description: '漢'.repeat(1_000) } as ToolDefinition]+ expect(heuristicCountTools(tools)).toBe(752 + 4)+ })})