Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): count message tokens by characters so minified tool results register (#731)

Alfredo Lopez4ed797a5a537
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index fb023890b..8ee570bfc 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.363",
+ "version": "0.1.364",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.363",
+ "version": "0.1.364",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 2fc9ce550..81e89ac35 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.363",
+ "version": "0.1.364",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts
index e1ca982ef..6b9e5880e 100644
--- a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts
+++ b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts
@@ -16,16 +16,19 @@
* directly rather than asserting it in a comment.
*/
import { describe, expect, it } from 'vitest'
-import { heuristicCount } from '../../core/tokenizer/heuristic'
-import type { ChatMessage } from '../../core/types'
import { minifiedMcpResult } from './minifiedMcpResult'
-const toolMessage = (content: string): ChatMessage => ({
- role: 'tool',
- content,
- tool_call_id: 'call_1',
- name: 'crm_search_contacts',
-})
+/**
+ * The word count `heuristicCount` performed before #731, reproduced here on
+ * purpose. This suite measures a property of the PAYLOAD, so it must not call
+ * the function under repair: doing so would turn F-1 red the moment the fix
+ * lands — exactly when it is supposed to still be green — and the suite would
+ * then be pinning the formula instead of the fixture.
+ */
+const wordTokens = (content: string): number => Math.floor(content.split(/\s+/).length * 1.3) + 4
+
+/** The chars/4 byte-pair-encoding approximation, likewise computed locally. */
+const byteTokens = (content: string): number => Math.ceil(content.length / 4) + 4
/** Prose of approximately `targetBytes` characters — the control payload. */
function proseOfSize(targetBytes: number): string {
@@ -60,20 +63,16 @@ describe('minifiedMcpResult (F-1 fixture witness)', () => {
it('F-1 exhibits the word/byte gap that #731 is about', () => {
const content = minifiedMcpResult(1, 33_000)
- const words = heuristicCount([toolMessage(content)])
- const bpe = Math.ceil(content.length / 4)
// The gap is the fixture's reason to exist: a word count of this payload is
// less than a third of its byte-derived size.
- expect(words * 3).toBeLessThan(bpe)
+ expect(wordTokens(content) * 3).toBeLessThan(byteTokens(content))
})
it('F-1 control: prose of the same size does NOT exhibit the gap', () => {
const content = proseOfSize(33_000)
- const words = heuristicCount([toolMessage(content)])
- const bpe = Math.ceil(content.length / 4)
- // Liveness witness for the assertion above: the same measurement on a prose
- // payload lands close to `chars / 4`, proving the previous case measures
- // the payload shape and not a constant of `heuristicCount`.
- expect(words * 3).toBeGreaterThan(bpe)
+ // Liveness witness for the assertion above: the same two measurements on a
+ // prose payload land close together, proving the previous case measures the
+ // payload's shape and not a constant of either formula.
+ expect(wordTokens(content) * 3).toBeGreaterThan(byteTokens(content))
})
})
diff --git a/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts b/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts
index 4726791e2..0df01445b 100644
--- a/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts
+++ b/mcp-host/src/core/extensions/__tests__/contextManager.antiThrash.test.ts
@@ -5,9 +5,10 @@
* plus the §8.3 Test A (defensive pending_approval guard does not mutate the
* compaction counter).
*
- * Fixtures rely on the heuristic token estimator (`heuristicCount`): floor(words×1.3)+4
- * per message. We build deterministic fixtures rather than mocking the estimator
- * so the assertions exercise the real wiring end-to-end.
+ * Fixtures rely on the heuristic token estimator (`heuristicCount`): ceil(chars/4)+4
+ * per message (#731; it was floor(words×1.3)+4 before). We build deterministic
+ * fixtures rather than mocking the estimator so the assertions exercise the real
+ * wiring end-to-end.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { Counter, register } from 'prom-client'
@@ -26,7 +27,7 @@ import {
clerumCompactionTotal,
} from '../contextManager'
-const TINY = 'x' // 1 word → floor(1×1.3)+4 = 5 tokens per message
+const TINY = 'x' // 1 char → ceil(1/4)+4 = 5 tokens per message
const PAD_WORDS = 600 // big enough that 3 of them dominate the histogram
/**
diff --git a/mcp-host/src/core/extensions/__tests__/prePrune.test.ts b/mcp-host/src/core/extensions/__tests__/prePrune.test.ts
index 6124d3528..e1ad7e253 100644
--- a/mcp-host/src/core/extensions/__tests__/prePrune.test.ts
+++ b/mcp-host/src/core/extensions/__tests__/prePrune.test.ts
@@ -112,7 +112,14 @@ describe('T1.2 prePrune — oneLineSummaries', () => {
})
it('skips already-deduped tool messages', () => {
- const longOutput = Array.from({ length: 210 }, () => 'x').join(' ')
+ // Sized against the 200-token threshold `oneLineSummaries` is called with
+ // below: 420 single-character words joined by spaces is 839 characters, so
+ // `heuristicCount` reads ceil(839/4)+4 = 214 tokens and the message is
+ // summarized. Recalibrated in #731 — under the previous word-count formula
+ // 210 words sufficed (floor(210×1.3)+4 = 277); under ceil(chars/4)+4 the
+ // same payload reads 109 and the summary never fires, which made this test
+ // assert a summary that legitimately had not happened.
+ const longOutput = Array.from({ length: 420 }, () => 'x').join(' ')
const messages: ChatMessage[] = [
assistantToolCall('tc_1', 'shell_exec', { command: 'npm test' }),
toolResult('tc_1', 'shell_exec', longOutput),
diff --git a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts
index d7f7249ce..28c58407d 100644
--- a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts
+++ b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts
@@ -8,20 +8,22 @@ describe('heuristicCount', () => {
expect(heuristicCount([])).toBe(0)
})
- it('counts floor(words*1.3)+4 per message', () => {
+ it('counts ceil(chars/4)+4 per message', () => {
const msgs: ChatMessage[] = [
- { role: 'user', content: 'one two three four five' }, // 5 words → 6 + 4 = 10
- { role: 'assistant', content: 'a b' }, // 2 words → 2 + 4 = 6
+ { role: 'user', content: 'one two three four five' }, // 23 chars → ceil(23/4)=6, +4 = 10
+ { role: 'assistant', content: 'a b' }, // 3 chars → ceil(3/4)=1, +4 = 5
]
- expect(heuristicCount(msgs)).toBe(16)
+ expect(heuristicCount(msgs)).toBe(15)
})
it('treats missing content as empty string', () => {
const msgs: ChatMessage[] = [
{ role: 'assistant', content: '', tool_calls: [{ id: 'x', name: 'y', arguments: {} }] },
]
- // '' splits into [''] (length 1) → floor(1.3)+4 = 5
- expect(heuristicCount(msgs)).toBe(5)
+ // '' → ceil(0/4)=0, +4 = 4. The framing overhead is all that remains, and
+ // the `tool_calls` payload is still uncounted here — A2 (#731, step 3) is
+ // what adds it.
+ expect(heuristicCount(msgs)).toBe(4)
})
it('T-A1 counts minified JSON by characters, not by whitespace-separated words', () => {
diff --git a/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts b/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts
index d0f9a2eed..ce66616c7 100644
--- a/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts
+++ b/mcp-host/src/core/tokenizer/fallbackTokenCounter.ts
@@ -3,7 +3,8 @@
* API or compatible tokenizer (ZAI, Bailian, and any future OpenAI-compatible
* shim with a custom backend).
*
- * Bias factor: 1.3× over the legacy word-count heuristic to over-estimate.
+ * Bias factor: 1.3× over the `ceil(chars / 4) + 4` heuristic to over-estimate
+ * (it biased the word-count heuristic the same way before #731).
* Empirically the heuristic mis-estimates by ±25-40% against actual provider
* usage; biasing high reduces overflow risk at the cost of slightly earlier
* compaction.
diff --git a/mcp-host/src/core/tokenizer/heuristic.ts b/mcp-host/src/core/tokenizer/heuristic.ts
index be7f8c995..a5ee7cd44 100644
--- a/mcp-host/src/core/tokenizer/heuristic.ts
+++ b/mcp-host/src/core/tokenizer/heuristic.ts
@@ -3,31 +3,41 @@
* implementations. Extracted from `core/conversation/compaction.ts` so the
* counters can use it without circular imports.
*
- * Formula (Risk 5.6 in the original code): `floor(word_count × 1.3) + 4` per
- * message. Nominally accurate to ±25%; significantly worse for tool-heavy
- * conversations because it ignores `tool_calls` payloads, `contentParts`,
- * and provider-specific framing overhead. Keep as the last-resort fallback
- * for offline mode and provider errors — never as the primary path.
+ * Formula: `ceil(chars / 4) + 4` per message — the standard byte-pair-encoding
+ * approximation, the same one `heuristicCountTools` below has always used.
+ *
+ * It replaced `floor(word_count × 1.3) + 4` in #731. A word count is a fair
+ * approximation for prose, where a space arrives every few characters, and a
+ * bad one for the payload that actually fills an agentic context: a tool result
+ * carrying minified JSON has almost no whitespace, so the word count collapses
+ * it into a handful of "words". Measured on a 33 KB MCP result: 1,982 tokens by
+ * words against 8,329 by characters. `PressureContextManager` read ~70% while
+ * real pressure was ~258%, stayed in `passthrough`, and the request was refused
+ * by the provider attempt contract as `request exceeds maxRequestBodyBytes`.
+ *
+ * Still a heuristic, and still the last-resort fallback for offline mode and
+ * provider errors — it ignores `contentParts` and provider-specific framing.
+ * Never the primary path where an exact tokenizer exists.
*/
import type { ChatMessage, ToolDefinition } from '../types'
export function heuristicCount(messages: ChatMessage[]): number {
let total = 0
for (const msg of messages) {
- const wordCount = (msg.content ?? '').split(/\s+/).length
- total += Math.floor(wordCount * 1.3) + 4
+ total += Math.ceil((msg.content ?? '').length / 4) + 4
}
return total
}
/**
* Estimate the token cost of tool schemas with a character-based heuristic
- * (`ceil(chars / 4) + 4` per tool). Unlike messages, tool schemas are dense
- * minified JSON with virtually no whitespace, so a word count collapses the
- * whole `parameters` object into a handful of "words" and underestimates by
- * roughly 4× (a 173-char schema → ~11 word-tokens vs ~44 real tokens). The
- * `chars / 4` ratio is the standard byte-pair-encoding approximation and is a
- * far better fit for this payload shape.
+ * (`ceil(chars / 4) + 4` per tool) — the same formula `heuristicCount` uses,
+ * which this function had to itself until #731. Tool schemas are dense minified
+ * JSON with virtually no whitespace, so a word count collapses the whole
+ * `parameters` object into a handful of "words" and underestimates by roughly
+ * 4× (a 173-char schema → ~11 word-tokens vs ~44 real tokens). The `chars / 4`
+ * ratio is the standard byte-pair-encoding approximation and is a far better
+ * fit for this payload shape — and, as #731 established, for tool results too.
*
* The estimate also covers the same surface OpenAI's exact tokenizer bills for
* a tool: `name` + `description` + `JSON.stringify(parameters)` — not just