Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): count message tokens by characters so minified tool results register (#731)
Alfredo Lopez4ed797a5a537
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex fb023890b..8ee570bfc 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.363",+ "version": "0.1.364","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.363",+ "version": "0.1.364","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 2fc9ce550..81e89ac35 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.363",+ "version": "0.1.364","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.tsindex e1ca982ef..6b9e5880e 100644--- a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts+++ b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts@@ -16,16 +16,19 @@* directly rather than asserting it in a comment.*/import { describe, expect, it } from 'vitest'-import { heuristicCount } from '../../core/tokenizer/heuristic'-import type { ChatMessage } from '../../core/types'import { minifiedMcpResult } from './minifiedMcpResult'-const toolMessage = (content: string): ChatMessage => ({- role: 'tool',- content,- tool_call_id: 'call_1',- name: 'crm_search_contacts',-})+/**+ * The word count `heuristicCount` performed before #731, reproduced here on+ * purpose. This suite measures a property of the PAYLOAD, so it must not call+ * the function under repair: doing so would turn F-1 red the moment the fix+ * lands — exactly when it is supposed to still be green — and the suite would+ * then be pinning the formula instead of the fixture.+ */+const wordTokens = (content: string): number => Math.floor(content.split(/\s+/).length * 1.3) + 4++/** The chars/4 byte-pair-encoding approximation, likewise computed locally. */+const byteTokens = (content: string): number => Math.ceil(content.length / 4) + 4/** Prose of approximately `targetBytes` characters — the control payload. */function proseOfSize(targetBytes: number): string {@@ -60,20 +63,16 @@ describe('minifiedMcpResult (F-1 fixture witness)', () => {it('F-1 exhibits the word/byte gap that #731 is about', () => {const content = minifiedMcpResult(1, 33_000)- const words = heuristicCount([toolMessage(content)])- const bpe = Math.ceil(content.length / 4)// The gap is the fixture's reason to exist: a word count of this payload is// less than a third of its byte-derived size.- expect(words * 3).toBeLessThan(bpe)+ expect(wordTokens(content) * 3).toBeLessThan(byteTokens(content))})it('F-1 control: prose of the same size does NOT exhibit the gap', () => {const content = proseOfSize(33_000)- const words = heuristicCount([toolMessage(content)])- const bpe = Math.ceil(content.length / 4)- // Liveness witness for the assertion above: the same measurement on a prose- // payload lands close to `chars / 4`, proving the previous case measures- // the payload shape and not a constant of `heuristicCount`.- expect(words * 3).toBeGreaterThan(bpe)+ // Liveness witness for the assertion above: the same two measurements on a+ // prose payload land close together, proving the previous case measures the+ // payload's shape and not a constant of either formula.+ expect(wordTokens(content) * 3).toBeGreaterThan(byteTokens(content))})})diff --git a/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts b/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.tsindex 4726791e2..0df01445b 100644--- a/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts+++ b/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts@@ -5,9 +5,10 @@* plus the §8.3 Test A (defensive pending_approval guard does not mutate the* compaction counter).*- * Fixtures rely on the heuristic token estimator (`heuristicCount`): floor(words×1.3)+4- * per message. We build deterministic fixtures rather than mocking the estimator- * so the assertions exercise the real wiring end-to-end.+ * Fixtures rely on the heuristic token estimator (`heuristicCount`): ceil(chars/4)+4+ * per message (#731; it was floor(words×1.3)+4 before). We build deterministic+ * fixtures rather than mocking the estimator so the assertions exercise the real+ * wiring end-to-end.*/import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { Counter, register } from 'prom-client'@@ -26,7 +27,7 @@ import {clerumCompactionTotal,} from '../contextManager'-const TINY = 'x' // 1 word → floor(1×1.3)+4 = 5 tokens per message+const TINY = 'x' // 1 char → ceil(1/4)+4 = 5 tokens per messageconst PAD_WORDS = 600 // big enough that 3 of them dominate the histogram/**diff --git a/mcp-host/src/core/extensions/__tests__/prePrune.test.ts b/mcp-host/src/core/extensions/__tests__/prePrune.test.tsindex 6124d3528..e1ad7e253 100644--- a/mcp-host/src/core/extensions/__tests__/prePrune.test.ts+++ b/mcp-host/src/core/extensions/__tests__/prePrune.test.ts@@ -112,7 +112,14 @@ describe('T1.2 prePrune — oneLineSummaries', () => {})it('skips already-deduped tool messages', () => {- const longOutput = Array.from({ length: 210 }, () => 'x').join(' ')+ // Sized against the 200-token threshold `oneLineSummaries` is called with+ // below: 420 single-character words joined by spaces is 839 characters, so+ // `heuristicCount` reads ceil(839/4)+4 = 214 tokens and the message is+ // summarized. Recalibrated in #731 — under the previous word-count formula+ // 210 words sufficed (floor(210×1.3)+4 = 277); under ceil(chars/4)+4 the+ // same payload reads 109 and the summary never fires, which made this test+ // assert a summary that legitimately had not happened.+ const longOutput = Array.from({ length: 420 }, () => 'x').join(' ')const messages: ChatMessage[] = [assistantToolCall('tc_1', 'shell_exec', { command: 'npm test' }),toolResult('tc_1', 'shell_exec', longOutput),diff --git a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.tsindex d7f7249ce..28c58407d 100644--- a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts+++ b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts@@ -8,20 +8,22 @@ describe('heuristicCount', () => {expect(heuristicCount([])).toBe(0)})- it('counts floor(words*1.3)+4 per message', () => {+ it('counts ceil(chars/4)+4 per message', () => {const msgs: ChatMessage[] = [- { role: 'user', content: 'one two three four five' }, // 5 words → 6 + 4 = 10- { role: 'assistant', content: 'a b' }, // 2 words → 2 + 4 = 6+ { role: 'user', content: 'one two three four five' }, // 23 chars → ceil(23/4)=6, +4 = 10+ { role: 'assistant', content: 'a b' }, // 3 chars → ceil(3/4)=1, +4 = 5]- expect(heuristicCount(msgs)).toBe(16)+ expect(heuristicCount(msgs)).toBe(15)})it('treats missing content as empty string', () => {const msgs: ChatMessage[] = [{ role: 'assistant', content: '', tool_calls: [{ id: 'x', name: 'y', arguments: {} }] },]- // '' splits into [''] (length 1) → floor(1.3)+4 = 5- expect(heuristicCount(msgs)).toBe(5)+ // '' → ceil(0/4)=0, +4 = 4. The framing overhead is all that remains, and+ // the `tool_calls` payload is still uncounted here — A2 (#731, step 3) is+ // what adds it.+ expect(heuristicCount(msgs)).toBe(4)})it('T-A1 counts minified JSON by characters, not by whitespace-separated words', () => {diff --git a/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts b/mcp-host/src/core/tokenizer/fallbackTokenCounter.tsindex d0f9a2eed..ce66616c7 100644--- a/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts+++ b/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts@@ -3,7 +3,8 @@* API or compatible tokenizer (ZAI, Bailian, and any future OpenAI-compatible* shim with a custom backend).*- * Bias factor: 1.3× over the legacy word-count heuristic to over-estimate.+ * Bias factor: 1.3× over the `ceil(chars / 4) + 4` heuristic to over-estimate+ * (it biased the word-count heuristic the same way before #731).* Empirically the heuristic mis-estimates by ±25-40% against actual provider* usage; biasing high reduces overflow risk at the cost of slightly earlier* compaction.diff --git a/mcp-host/src/core/tokenizer/heuristic.ts b/mcp-host/src/core/tokenizer/heuristic.tsindex be7f8c995..a5ee7cd44 100644--- a/mcp-host/src/core/tokenizer/heuristic.ts+++ b/mcp-host/src/core/tokenizer/heuristic.ts@@ -3,31 +3,41 @@* implementations. Extracted from `core/conversation/compaction.ts` so the* counters can use it without circular imports.*- * Formula (Risk 5.6 in the original code): `floor(word_count × 1.3) + 4` per- * message. Nominally accurate to ±25%; significantly worse for tool-heavy- * conversations because it ignores `tool_calls` payloads, `contentParts`,- * and provider-specific framing overhead. Keep as the last-resort fallback- * for offline mode and provider errors — never as the primary path.+ * Formula: `ceil(chars / 4) + 4` per message — the standard byte-pair-encoding+ * approximation, the same one `heuristicCountTools` below has always used.+ *+ * It replaced `floor(word_count × 1.3) + 4` in #731. A word count is a fair+ * approximation for prose, where a space arrives every few characters, and a+ * bad one for the payload that actually fills an agentic context: a tool result+ * carrying minified JSON has almost no whitespace, so the word count collapses+ * it into a handful of "words". Measured on a 33 KB MCP result: 1,982 tokens by+ * words against 8,329 by characters. `PressureContextManager` read ~70% while+ * real pressure was ~258%, stayed in `passthrough`, and the request was refused+ * by the provider attempt contract as `request exceeds maxRequestBodyBytes`.+ *+ * Still a heuristic, and still the last-resort fallback for offline mode and+ * provider errors — it ignores `contentParts` and provider-specific framing.+ * Never the primary path where an exact tokenizer exists.*/import type { ChatMessage, ToolDefinition } from '../types'export function heuristicCount(messages: ChatMessage[]): number {let total = 0for (const msg of messages) {- const wordCount = (msg.content ?? '').split(/\s+/).length- total += Math.floor(wordCount * 1.3) + 4+ total += Math.ceil((msg.content ?? '').length / 4) + 4}return total}/*** Estimate the token cost of tool schemas with a character-based heuristic- * (`ceil(chars / 4) + 4` per tool). Unlike messages, tool schemas are dense- * minified JSON with virtually no whitespace, so a word count collapses the- * whole `parameters` object into a handful of "words" and underestimates by- * roughly 4× (a 173-char schema → ~11 word-tokens vs ~44 real tokens). The- * `chars / 4` ratio is the standard byte-pair-encoding approximation and is a- * far better fit for this payload shape.+ * (`ceil(chars / 4) + 4` per tool) — the same formula `heuristicCount` uses,+ * which this function had to itself until #731. Tool schemas are dense minified+ * JSON with virtually no whitespace, so a word count collapses the whole+ * `parameters` object into a handful of "words" and underestimates by roughly+ * 4× (a 173-char schema → ~11 word-tokens vs ~44 real tokens). The `chars / 4`+ * ratio is the standard byte-pair-encoding approximation and is a far better+ * fit for this payload shape — and, as #731 established, for tool results too.** The estimate also covers the same surface OpenAI's exact tokenizer bills for* a tool: `name` + `description` + `JSON.stringify(parameters)` — not just