Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(mcp-host): pin the chars/4 gauge and the compaction journey for minified tool results (#731)
Alfredo Lopez46b7a7901e82
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 23bfbfa52..fb023890b 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.362",+ "version": "0.1.363","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.362",+ "version": "0.1.363","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex f91b6d21e..2fc9ce550 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.362",+ "version": "0.1.363","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.tsnew file mode 100644index 000000000..e1ca982ef--- /dev/null+++ b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.test.ts@@ -0,0 +1,79 @@+/**+ * F-1 — fixture witness for #731.+ *+ * These cases pass today and must keep passing after the fix; they assert+ * nothing about `heuristicCount`'s formula. Their job is to pin the two+ * properties every #731 test depends on, so the fixture cannot be replaced by+ * something that makes those tests green without a fix:+ *+ * 1. the fixture is the size it was asked for (±10%), and+ * 2. it exhibits the word/byte gap — a word count of the payload is far+ * below its `chars / 4` size.+ *+ * Property 2 is the substitution guard. A prose payload of identical byte size+ * has roughly one word per 6 characters, so its word count lands near+ * `chars / 6` and the gap collapses; the control case below measures that+ * directly rather than asserting it in a comment.+ */+import { describe, expect, it } from 'vitest'+import { heuristicCount } from '../../core/tokenizer/heuristic'+import type { ChatMessage } from '../../core/types'+import { minifiedMcpResult } from './minifiedMcpResult'++const toolMessage = (content: string): ChatMessage => ({+ role: 'tool',+ content,+ tool_call_id: 'call_1',+ name: 'crm_search_contacts',+})++/** Prose of approximately `targetBytes` characters — the control payload. */+function proseOfSize(targetBytes: number): string {+ const sentence =+ 'The assistant summarized the contact list and explained which records were relevant. '+ return sentence.repeat(Math.ceil(targetBytes / sentence.length)).slice(0, targetBytes)+}++describe('minifiedMcpResult (F-1 fixture witness)', () => {+ it('F-1 is deterministic for a given seed and size', () => {+ expect(minifiedMcpResult(7, 12_000)).toBe(minifiedMcpResult(7, 12_000))+ expect(minifiedMcpResult(8, 12_000)).not.toBe(minifiedMcpResult(7, 12_000))+ })++ it('F-1 reaches the requested size without overshooting by more than 10%', () => {+ for (const target of [12_000, 33_000, 35_000]) {+ const content = minifiedMcpResult(1, target)+ expect(content.length).toBeGreaterThanOrEqual(target)+ expect(content.length).toBeLessThanOrEqual(Math.ceil(target * 1.1))+ }+ })++ it('F-1 is minified: identical to its own canonical serialization', () => {+ const content = minifiedMcpResult(3, 12_000)+ // `JSON.stringify` emits no structural whitespace, so a payload equal to+ // the re-serialization of its own parse carries none either. The spaces+ // that remain are inside string values (names, titles, sentences), which is+ // what a real tool result looks like.+ expect(JSON.stringify(JSON.parse(content))).toBe(content)+ expect(content).not.toContain('\n')+ })++ it('F-1 exhibits the word/byte gap that #731 is about', () => {+ const content = minifiedMcpResult(1, 33_000)+ const words = heuristicCount([toolMessage(content)])+ const bpe = Math.ceil(content.length / 4)+ // The gap is the fixture's reason to exist: a word count of this payload is+ // less than a third of its byte-derived size.+ expect(words * 3).toBeLessThan(bpe)+ })++ it('F-1 control: prose of the same size does NOT exhibit the gap', () => {+ const content = proseOfSize(33_000)+ const words = heuristicCount([toolMessage(content)])+ const bpe = Math.ceil(content.length / 4)+ // Liveness witness for the assertion above: the same measurement on a prose+ // payload lands close to `chars / 4`, proving the previous case measures+ // the payload shape and not a constant of `heuristicCount`.+ expect(words * 3).toBeGreaterThan(bpe)+ })+})diff --git a/mcp-host/src/__tests__/fixtures/minifiedMcpResult.ts b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.tsnew file mode 100644index 000000000..610136de7--- /dev/null+++ b/mcp-host/src/__tests__/fixtures/minifiedMcpResult.ts@@ -0,0 +1,120 @@+/**+ * Deterministic fixture for #731 — a minified JSON tool result of a requested+ * byte size, shaped like what an MCP server actually returns (contact records+ * from a CRM/outreach integration).+ *+ * The payload shape is the point, not the size. A word count collapses dense+ * minified JSON into a handful of "words" because there is no whitespace to+ * split on, which is the defect `heuristicCount` carries and+ * `heuristicCountTools` already documents (`core/tokenizer/heuristic.ts:24-38`).+ * A prose fixture of the same byte size does NOT reproduce it: prose carries a+ * space every few characters, so its word count tracks its byte count and the+ * gauge reads it correctly.+ *+ * `minifiedMcpResult.test.ts` (F-1) pins that gap, so substituting a prose+ * fixture later fails loudly instead of silently turning the #731 tests green+ * without a fix.+ *+ * Deterministic by construction: a 32-bit LCG seeded from `seed`, no+ * `Math.random`, no clock. The same (seed, targetBytes) always yields the same+ * string.+ */++/** Numerical Recipes LCG — fixed constants, no global state. */+function lcg(seed: number): () => number {+ let state = seed >>> 0+ return () => {+ state = (Math.imul(state, 1664525) + 1013904223) >>> 0+ return state / 0x100000000+ }+}++const FIRST_NAMES = ['Joaquin', 'Marta', 'Andrei', 'Priya', 'Tomas', 'Chiara', 'Noor', 'Diego']+const LAST_NAMES = ['La Madrid', 'Neira', 'Okonkwo', 'Vasquez', 'Lindqvist', 'Haddad', 'Rossi']+const TITLES = [+ 'Chief Business Officer',+ 'Head of Partnerships',+ 'VP Revenue Operations',+ 'Director of Growth',+ 'Founder',+]+const COMPANIES = ['Not a Bot Agency', 'Northwind Labs', 'Veridian Systems', 'Halcyon Data']+const CITIES = ['Alicante', 'Tallinn', 'Lisbon', 'Bristol', 'Valparaiso']+const SENTIMENTS = ['positive', 'neutral', 'requested_followup']++const pick = <T>(rnd: () => number, xs: readonly T[]): T => xs[Math.floor(rnd() * xs.length)]++/**+ * One contact record. Every field is drawn from the LCG, so record `n` of a+ * given seed is stable across runs and machines.+ */+function contact(rnd: () => number, index: number): Record<string, unknown> {+ const first = pick(rnd, FIRST_NAMES)+ const last = pick(rnd, LAST_NAMES)+ const company = pick(rnd, COMPANIES)+ return {+ id: `c_${index}_${Math.floor(rnd() * 1e9).toString(36)}`,+ firstName: first,+ lastName: last,+ headline: `${pick(rnd, TITLES)} at ${company}`,+ linkedinUrl: `https://www.linkedin.com/in/${first.toLowerCase()}-${last+ .toLowerCase()+ .replace(/ /g, '-')}-${index}`,+ company: {+ name: company,+ domain: `${company.toLowerCase().replace(/[^a-z]/g, '')}.example`,+ location: `${pick(rnd, CITIES)}, ES`,+ employeeCount: 3 + Math.floor(rnd() * 900),+ },+ lastReply: {+ campaignId: `camp_${index}`,+ campaignName: 'Q3 Outbound - SaaS Partners',+ repliedAt: `2026-09-${String(1 + (index % 28)).padStart(2, '0')}T10:22:03.000Z`,+ messageText:+ 'Thanks for reaching out, this is relevant for us. Can we book a call next week?',+ sentiment: pick(rnd, SENTIMENTS),+ },+ tags: ['saas', 'partner', `tier-${1 + (index % 3)}`],+ score: Number((rnd() * 100).toFixed(4)),+ }+}++/**+ * A minified JSON tool result of at least `targetBytes` characters.+ *+ * Records are serialized one at a time and the envelope length is computed+ * exactly at each step, so the result overshoots the target by at most one+ * record (~400 characters). F-1 pins the overshoot at +10%.+ */+export function minifiedMcpResult(seed: number, targetBytes: number): string {+ if (!Number.isInteger(seed)) {+ throw new Error(`minifiedMcpResult: seed must be an integer, got ${seed}`)+ }+ if (!Number.isInteger(targetBytes) || targetBytes <= 0) {+ throw new Error(`minifiedMcpResult: targetBytes must be a positive integer, got ${targetBytes}`)+ }++ const rnd = lcg(seed)+ const records: string[] = []+ let recordBytes = 0+ let index = 0++ // Each record is already canonical minified JSON, so the envelope length is+ // arithmetic — no quadratic re-serialization of the growing array.+ const envelope = (parts: string[]): string =>+ `{"contacts":[${parts.join(',')}],"total":${parts.length}}`+ const projectedLength = (): number =>+ OPEN.length + recordBytes + Math.max(records.length - 1, 0) + CLOSE(records.length).length++ for (;;) {+ const record = JSON.stringify(contact(rnd, index++))+ records.push(record)+ recordBytes += record.length+ if (projectedLength() >= targetBytes) break+ }++ return envelope(records)+}++const OPEN = '{"contacts":['+const CLOSE = (n: number): string => `],"total":${n}}`diff --git a/mcp-host/src/core/extensions/__tests__/contextManager.test.ts b/mcp-host/src/core/extensions/__tests__/contextManager.test.tsindex 4fc951faf..c8009f2b3 100644--- a/mcp-host/src/core/extensions/__tests__/contextManager.test.ts+++ b/mcp-host/src/core/extensions/__tests__/contextManager.test.ts@@ -1,11 +1,17 @@-import { describe, expect, it, vi } from 'vitest'+import { beforeEach, describe, expect, it, vi } from 'vitest'+import { type Counter, register } from 'prom-client'+import { minifiedMcpResult } from '../../../__tests__/fixtures/minifiedMcpResult'import type { WorkspaceService } from '../../../workspace/service'import { makeFakeConversation } from '../../conversation/__testing__/makeFakeConversation'import { estimateTokens, splitTurns } from '../../conversation/compaction'import type { LlmPort } from '../../interfaces'import { validateToolLinkages } from '../../orchestration/toolUseLoop'import type { ChatMessage } from '../../types'-import { InLoopContextManager, PressureContextManager } from '../contextManager'+import {+ InLoopContextManager,+ PressureContextManager,+ clerumCompactionTotal,+} from '../contextManager'/*** Generate a message array with roughly the specified number of tokens.@@ -418,3 +424,84 @@ describe('PressureContextManager - archived markdown', () => {expect(markdown).not.toContain(longContent) // full content not present})})++/**+ * T-A3 — #731. The deployed decision path for `codex-subscription` and+ * `grok-subscription`: a `PressureContextManager` with NO token counter, so+ * `computePressure` falls through to `estimateTokens` → `heuristicCount`+ * (`contextManager.ts:469-471`). Both providers map to `FallbackTokenCounter`+ * (`llm/registryCore.ts:203-204`), which is the same word heuristic with a 1.3+ * bias, so no configuration makes this history compact today.+ *+ * prom-client metrics are process-global, so this block resets the registry+ * before each case. No other case in this file reads a counter.+ */+describe('PressureContextManager — #731 MCP-heavy history', () => {+ /** Snapshot a counter's labeled value (0 when the series does not exist yet). */+ async function counterValue(+ metric: Counter<string>,+ labels: Record<string, string>+ ): Promise<number> {+ const data = await metric.get()+ for (const v of data.values) {+ if (Object.entries(labels).every(([k, val]) => v.labels[k] === val)) return v.value+ }+ return 0+ }++ /**+ * The shape an agentic MCP session produces: a user request, an assistant+ * message carrying one tool call with small arguments, and a tool result+ * carrying a large minified JSON payload.+ */+ function mcpHeavyHistory(turns: number, bytesPerResult: number): ChatMessage[] {+ const msgs: ChatMessage[] = [{ role: 'system', content: 'You are a helpful assistant.' }]+ for (let turn = 1; turn <= turns; turn++) {+ msgs.push({ role: 'user', content: `Find the contacts that replied in campaign ${turn}.` })+ msgs.push({+ role: 'assistant',+ content: '',+ tool_calls: [+ {+ id: `call_${turn}`,+ name: 'crm_search_contacts',+ arguments: { campaignId: `camp_${turn}`, limit: 60 },+ },+ ],+ })+ msgs.push({+ role: 'tool',+ content: minifiedMcpResult(turn, bytesPerResult),+ tool_call_id: `call_${turn}`,+ name: 'crm_search_contacts',+ })+ }+ return msgs+ }++ beforeEach(() => {+ register.resetMetrics()+ })++ it('T-A3 compacts a realistic MCP-heavy history under the default budget (#731)', async () => {+ const msgs = mcpHeavyHistory(12, 35_000)+ const manager = new PressureContextManager(100000)++ const result = await manager.manage(msgs, makeFakeConversation())++ // Route: a tier ran, and it was the emergency tier.+ expect(result).not.toBe(msgs)+ expect(await counterValue(clerumCompactionTotal, { tier: 'truncate', outcome: 'ok' })).toBe(1)++ // State: the history shrank and the tool linkages survived the cut.+ expect(result.length).toBeLessThan(msgs.length)+ expect(() => validateToolLinkages(result)).not.toThrow()++ // Business signal: the tier removed BYTES, not just array entries. A cut that+ // dropped only the short user turns would satisfy the length assertion above+ // while leaving the request exactly as oversized as before.+ expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan(+ Buffer.byteLength(JSON.stringify(msgs)) / 2+ )+ })+})diff --git a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.tsindex f54769d7d..d7f7249ce 100644--- a/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts+++ b/mcp-host/src/core/tokenizer/__tests__/heuristic.test.ts@@ -1,4 +1,5 @@import { describe, expect, it } from 'vitest'+import { minifiedMcpResult } from '../../../__tests__/fixtures/minifiedMcpResult'import type { ChatMessage, ToolDefinition } from '../../types'import { heuristicCount, heuristicCountTools } from '../heuristic'@@ -22,6 +23,21 @@ describe('heuristicCount', () => {// '' splits into [''] (length 1) → floor(1.3)+4 = 5expect(heuristicCount(msgs)).toBe(5)})++ it('T-A1 counts minified JSON by characters, not by whitespace-separated words', () => {+ // #731 — a tool result carrying dense minified JSON is the payload shape+ // that breaks a word count. `heuristicCountTools` already applies+ // `ceil(chars / 4)` for exactly this reason (see its comment); messages,+ // where tool RESULTS live, never got the same correction.+ const content = minifiedMcpResult(1, 33_000)+ const msg: ChatMessage = {+ role: 'tool',+ content,+ tool_call_id: 'call_1',+ name: 'crm_search_contacts',+ }+ expect(heuristicCount([msg])).toBeGreaterThanOrEqual(Math.ceil(content.length / 4))+ })})describe('heuristicCountTools', () => {diff --git a/mcp-host/src/llm/__tests__/contextOverrunJourney.test.ts b/mcp-host/src/llm/__tests__/contextOverrunJourney.test.tsnew file mode 100644index 000000000..e2f7ef943--- /dev/null+++ b/mcp-host/src/llm/__tests__/contextOverrunJourney.test.ts@@ -0,0 +1,149 @@+/**+ * #731 — the journey tests.+ *+ * These prove the route, not the destination: a long MCP-heavy conversation is+ * compacted by `PressureContextManager` and then dispatched through+ * `CodexSubscriptionProvider`, which hashes the canonical request and refuses it+ * when the attempt contract says it is too large. Every intermediate transition+ * is asserted — the tier that ran, the shape of the history that came out, and+ * the request the provider actually built — because a test that only asserts the+ * final state would pass just as happily if compaction never ran and the request+ * happened to fit.+ *+ * J1 is the success route (this step). J2, the failure route, arrives in step 4+ * of the plan together with the error-taxonomy fix.+ */+import { beforeEach, describe, expect, it, vi } from 'vitest'+import { type Counter, register } from 'prom-client'+import { hashCanonicalCodexRequest } from '@clerum/llm-provider-attempt-contract'+import { minifiedMcpResult } from '../../__tests__/fixtures/minifiedMcpResult'+import { makeFakeConversation } from '../../core/conversation/__testing__/makeFakeConversation'+import { PressureContextManager, clerumCompactionTotal } from '../../core/extensions/contextManager'+import { validateToolLinkages } from '../../core/orchestration/toolUseLoop'+import type { ChatMessage, ToolDefinition } from '../../core/types'+import { CodexSubscriptionProvider } from '../codexSubscription'++const requestHash = 'a'.repeat(64)++/**+ * The provider's dependencies, with `authorize` and `stream` as spies. Same+ * shape as the `deps()` helper in `codexSubscription.test.ts:21-47`; replicated+ * here rather than exported from that file so neither test file constrains the+ * other's fixtures.+ */+function deps() {+ const authorize = vi.fn().mockResolvedValue({+ providerAttemptId: 'attempt-1',+ requestHash,+ executionTicket: 'ticket-123456',+ expiresAt: '2026-08-20T10:00:00.000Z',+ })+ const stream = vi.fn().mockResolvedValue({+ text: 'hello from proxy',+ toolCalls: [],+ outcome: 'success',+ })+ return {+ authorizer: { authorize },+ proxy: { stream },+ attemptContext: vi.fn(() => ({+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ hostRef: 'chatllm',+ })),+ authorize,+ stream,+ }+}++const TOOLS: ToolDefinition[] = [+ {+ name: 'crm_search_contacts',+ description: 'Search contacts that replied to a campaign.',+ parameters: { type: 'object', properties: { campaignId: { type: 'string' } } },+ },+]++/** Snapshot a counter's labeled value (0 when the series does not exist yet). */+async function counterValue(+ metric: Counter<string>,+ labels: Record<string, string>+): Promise<number> {+ const data = await metric.get()+ for (const v of data.values) {+ if (Object.entries(labels).every(([k, val]) => v.labels[k] === val)) return v.value+ }+ return 0+}++/**+ * `turns` rounds of user → assistant (one tool call, small arguments) → tool+ * result carrying `bytesPerResult` of minified JSON.+ */+function mcpHeavyHistory(turns: number, bytesPerResult: number): ChatMessage[] {+ const msgs: ChatMessage[] = [{ role: 'system', content: 'You are a helpful assistant.' }]+ for (let turn = 1; turn <= turns; turn++) {+ msgs.push({ role: 'user', content: `Find the contacts that replied in campaign ${turn}.` })+ msgs.push({+ role: 'assistant',+ content: '',+ tool_calls: [+ {+ id: `call_${turn}`,+ name: 'crm_search_contacts',+ arguments: { campaignId: `camp_${turn}`, limit: 60 },+ },+ ],+ })+ msgs.push({+ role: 'tool',+ content: minifiedMcpResult(turn, bytesPerResult),+ tool_call_id: `call_${turn}`,+ name: 'crm_search_contacts',+ })+ }+ return msgs+}++describe('#731 context-overrun journey', () => {+ beforeEach(() => {+ register.resetMetrics()+ })++ it('J1 an MCP-heavy multi-turn history compacts and the contract accepts the request (#731)', async () => {+ const msgs = mcpHeavyHistory(32, 35_000)+ // Precondition, asserted rather than assumed: the uncompacted history is+ // already past the contract's 1 MiB ceiling, so a request built from it+ // cannot be accepted. Without this the rest of the test could pass on a+ // history that never needed compacting.+ expect(Buffer.byteLength(JSON.stringify(msgs), 'utf8')).toBeGreaterThan(1_048_576)++ const manager = new PressureContextManager(100000)+ const conversation = makeFakeConversation()+ const wired = deps()+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', wired as never)++ const managed = await manager.manage(msgs, conversation)++ // Route: the emergency tier ran, and the backoff did not — those are the two+ // paths that can return a shorter or an unchanged array respectively, and+ // only one of them is this journey.+ expect(await counterValue(clerumCompactionTotal, { tier: 'truncate', outcome: 'ok' })).toBe(1)+ expect(+ await counterValue(clerumCompactionTotal, { tier: 'truncate', outcome: 'thrashing' })+ ).toBe(0)++ // State: shorter history, linkages intact.+ expect(managed.length).toBeLessThan(msgs.length)+ expect(() => validateToolLinkages(managed)).not.toThrow()++ await provider.completeSingleTurnWithTools(managed, TOOLS)++ // Business signal: the provider got far enough to authorize, and what it+ // built is a request the contract accepts and would hash.+ expect(wired.authorize).toHaveBeenCalledTimes(1)+ const req = wired.authorize.mock.calls[0][0].request+ expect(Buffer.byteLength(JSON.stringify(req), 'utf8')).toBeLessThanOrEqual(1_048_576)+ expect(hashCanonicalCodexRequest(req).ok).toBe(true)+ })+})