Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore(mcp-host): checkpoint attachment work before dev sync
Alfredo Lopezc1dfe8950a56
diff --git a/mcp-host/README.md b/mcp-host/README.mdindex aaf2e6acc..02d517c1a 100644--- a/mcp-host/README.md+++ b/mcp-host/README.md@@ -17,51 +17,51 @@ MCP Host is a Kubernetes-native service that reads Host CRD configuration and pr### Environment Variables-| Variable | Explanation | Canonical example |-| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |-| `CLERUM_DEV_MODE` | Enables local/dev bootstrap mode that bypasses Host CRD discovery and reads host/runtime config from env. | `CLERUM_DEV_MODE=true` |-| `CLERUM_HOST_NAME` | Selects which Host CRD to load in production mode; in non-production fallback it defaults to `dev-host`. | `CLERUM_HOST_NAME=chatllm` |-| `CLERUM_NAMESPACE` | Sets the Kubernetes namespace used for Host and Secret lookup when running in cluster mode. | `CLERUM_NAMESPACE=mcp-host` |-| `CLERUM_SERVER_PORT` | Sets the HTTP listen port for the runtime API server. | `CLERUM_SERVER_PORT=8080` |-| `CLERUM_CONTEXT_MAPPER_URL` | Points to the host-context-controller endpoint used to resolve MCP servers for a context. | `CLERUM_CONTEXT_MAPPER_URL=http://host-context-controller-api-gateway.control-plane.svc.cluster.local:8081` |-| `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL` | Controls polling interval (milliseconds) for MCP server config changes from context mapper. | `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL=30000` |-| `HCC_AUTHORITY_MAX_STALENESS_MS` | Maximum time a previously published MCP fleet may survive an unavailable HCC (default 180000 ms; hard-capped at 600000 ms). It must be greater than `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL` in cluster mode; invalid combinations fail startup. Minikube pins 60000 ms for its deterministic staleness E2E. | `HCC_AUTHORITY_MAX_STALENESS_MS=180000` |-| `CLERUM_HOST_CONFIG` | Provides a full Host spec as JSON in dev mode, overriding provider/model context env composition. | `CLERUM_HOST_CONFIG={"host":"dev-host","contextRef":"dev-context","secretRef":"dev-secret","model":{"provider":"openai","name":"gpt-5.4-mini"}}` |-| `CLERUM_MODEL_PROVIDER` | Chooses the model provider in dev mode when `CLERUM_HOST_CONFIG` is not set. | `CLERUM_MODEL_PROVIDER=openai` |-| `CLERUM_MODEL_NAME` | Overrides the default provider model name in dev mode. Used only when `CLERUM_MODEL_PROVIDER` is also set; an auto-detected provider keeps its default model. | `CLERUM_MODEL_NAME=gpt-5.4-mini` |-| `CLERUM_MCP_SERVERS` | Supplies dev MCP server definitions as JSON so you can run without context mapper. Each entry needs `name`, `contextRef`, `transport`, `enabled`, and a `status` object — a server whose `status.ready` is not `true` is parsed but skipped as "not ready". | `CLERUM_MCP_SERVERS=[{"name":"filesystem","contextRef":"dev-context","transport":{"type":"streamableHttp","url":"http://localhost:3101"},"enabled":true,"status":{"deployed":true,"ready":true}}]` |-| `CLERUM_MCP_TOOL_TIMEOUT_MS` | Per-call timeout passed to MCP SDK `Client.callTool(...)` requests. The effective timeout is still bounded by any caller workflow or snippet step budget. | `CLERUM_MCP_TOOL_TIMEOUT_MS=1500000` |-| `CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS` | Maximum total MCP SDK tool-call budget used at the SDK request boundary. Operators may tune it up to the existing workflow step ceiling; invalid values fail closed at runtime. | `CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS=1500000` |-| `OPENAI_API_KEY` | OpenAI API key used by dev mode when provider selection resolves to OpenAI. | `OPENAI_API_KEY=sk-xxxx` |-| `CLAUDE_API_KEY` | Anthropic API key used by dev mode when provider selection resolves to Claude. | `CLAUDE_API_KEY=sk-ant-xxxx` |-| `ZAI_API_KEY` | ZAI API key used by dev mode when provider selection resolves to ZAI. | `ZAI_API_KEY=zai-xxxx` |-| `BAILIAN_API_KEY` | Bailian API key used by dev mode when provider selection resolves to Bailian/Qwen. | `BAILIAN_API_KEY=ba-xxxx` |-| `CLERUM_AGENT_TASK_DELAY` | Minimum interval in milliseconds between task dispatches; does not delay individual tool iterations. | `CLERUM_AGENT_TASK_DELAY=3` |-| `CLERUM_AGENT_MAX_TASK_DURATION` | Maximum active task execution time in milliseconds (24 hours), accumulated across approval resumptions. Approval waiting is excluded. | `CLERUM_AGENT_MAX_TASK_DURATION=86400000` |-| `CLERUM_AGENT_MAX_TOOL_CALLS` | Maximum LLM/tool iterations per task; each iteration may call multiple tools. Configured per Host process. | `CLERUM_AGENT_MAX_TOOL_CALLS=1000` |-| `CLERUM_AGENT_MAX_QUEUE_SIZE` | Maximum pending queue size before new tasks are rejected/back-pressured. | `CLERUM_AGENT_MAX_QUEUE_SIZE=100` |-| `CLERUM_ENABLE_APPROVAL` | Enables the approval gate workflow for tool execution decisions. | `CLERUM_ENABLE_APPROVAL=true` |-| `CLERUM_APPROVAL_CONFIG` | JSON config for approval policy/channels when not sourced from Host CRD. | `CLERUM_APPROVAL_CONFIG={"defaultPolicy":"designated_approvers","channels":{"telegram":{"enabled":true,"approvers":["123456"]}}}` |-| `CLERUM_ENABLE_NUDGE` | Enables the nudge controller that pushes model/tool retry guidance when output is rejected. | `CLERUM_ENABLE_NUDGE=false` |-| `CLERUM_NUDGE_MAX_ITERATIONS` | Maximum number of nudge cycles allowed for a task. | `CLERUM_NUDGE_MAX_ITERATIONS=3` |-| `CLERUM_CONTEXT_MAX_TOKENS` | Token budget used by context compaction when the model's allowlist row carries no context window. `codex-subscription` and `grok-subscription` models use 256000 instead; the Host logs `context_window_resolved` with the window and its source. | `CLERUM_CONTEXT_MAX_TOKENS=100000` |-| `CLERUM_COMPACTION_PRE_PRUNE` | Master switch for pre-pruning the history (deduplication, one-lining, JSON truncation, media stripping). It runs only once the history reaches the compaction threshold (80% of the context window), before any compaction tier is chosen, and at history rehydration under the same threshold. On by default; set to `false` to disable every pass at once. | `CLERUM_COMPACTION_PRE_PRUNE=true` |-| `CLERUM_WORKSPACE_PATH` | Base workspace path used by native tools and workspace-aware operations. | `CLERUM_WORKSPACE_PATH=/workspace` |-| `CLERUM_SHELL_TIMEOUT` | Maximum execution time in milliseconds for shell native-tool commands. | `CLERUM_SHELL_TIMEOUT=1500000` |-| `CLERUM_TOOL_TIMEOUT` | Maximum execution time in milliseconds for native tool wrapper calls. This is separate from the MCP SDK request timeout controlled by `CLERUM_MCP_TOOL_TIMEOUT_MS`. | `CLERUM_TOOL_TIMEOUT=1500000` |-| `CLERUM_HTTP_ALLOWLIST` | Comma-separated allowlist of outbound hosts/URLs allowed for HTTP native tools. With an allowlist set, you can also skip per-call approval by adding `tools: { http_request: false }` under `Host.spec.approval` (see Host CRD). | `CLERUM_HTTP_ALLOWLIST=api.github.com,example.com` |-| `CLERUM_ENV_ALLOWLIST` | Comma-separated list of environment variables forwarded into native tool execution contexts. | `CLERUM_ENV_ALLOWLIST=PATH,HOME,USER,SHELL,LANG,TERM` |-| `CLERUM_MEMORY_MAX_SIZE` | Maximum bytes reserved for in-memory native tool buffers/state. | `CLERUM_MEMORY_MAX_SIZE=1048576` |-| `CLERUM_ENABLE_RESPONSE_ATTACHMENTS` | Enables attachment passthrough in runtime responses when tools return supported media. | `CLERUM_ENABLE_RESPONSE_ATTACHMENTS=true` |-| `CLERUM_ATTACHMENT_MAX_COUNT` | Maximum number of attachments returned per response. | `CLERUM_ATTACHMENT_MAX_COUNT=3` |-| `CLERUM_ATTACHMENT_MAX_BYTES` | Maximum decoded size per attachment in bytes (oversized payloads are dropped). | `CLERUM_ATTACHMENT_MAX_BYTES=52428800` |-| `CLERUM_ATTACHMENT_FILE_MAX_BYTES` | Maximum decoded size in bytes of one incoming `kind:'file'` attachment; one larger file rejects the message with `FILE_ATTACHMENT_TOO_LARGE`. File base64 has its own 16 MiB quota in the chat body (11 MiB per file at the default), and the message text stays in the 6 MiB non-image share; a lower value here does not lower that quota. An invalid value stops the Host at startup. | `CLERUM_ATTACHMENT_FILE_MAX_BYTES=11534336` |-| `CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES` | Maximum bytes one `clerum__attachment_read` call returns, and its default page; longer text is returned with `truncated:true` and read on with `offset`. The page is shipped to the model inline (the tool is exempt from `CLERUM_TOOL_SPILLOVER_THRESHOLD`), so size it against `CLERUM_CONTEXT_MAX_TOKENS`: the default is ~16k heuristic tokens, about a fifth of the default compaction budget. An invalid value stops the Host at startup. | `CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES=65536` |-| `CLERUM_MEMORY_ENABLED` | Enables workspace memory persistence features. | `CLERUM_MEMORY_ENABLED=true` |-| `CLERUM_MEMORY_WORKSPACE_PATH` | Sets workspace directory path used when memory mode is enabled. | `CLERUM_MEMORY_WORKSPACE_PATH=/workspace` |-| `CLERUM_PERSONALIZATION_ENABLED` | Enables identity/personalization seeding behavior. | `CLERUM_PERSONALIZATION_ENABLED=true` |-| `CLERUM_IDENTITY_SEED` | JSON identity seed used to initialize personalization artifacts when enabled. | `CLERUM_IDENTITY_SEED={"displayName":"Clerum Agent","tone":"concise"}` |-| `NODE_ENV` | Influences production checks and defaults (for example strict host-name requirement in production). | `NODE_ENV=production` |+| Variable | Explanation | Canonical example |+| --------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `CLERUM_DEV_MODE` | Enables local/dev bootstrap mode that bypasses Host CRD discovery and reads host/runtime config from env. | `CLERUM_DEV_MODE=true` |+| `CLERUM_HOST_NAME` | Selects which Host CRD to load in production mode; in non-production fallback it defaults to `dev-host`. | `CLERUM_HOST_NAME=chatllm` |+| `CLERUM_NAMESPACE` | Sets the Kubernetes namespace used for Host and Secret lookup when running in cluster mode. | `CLERUM_NAMESPACE=mcp-host` |+| `CLERUM_SERVER_PORT` | Sets the HTTP listen port for the runtime API server. | `CLERUM_SERVER_PORT=8080` |+| `CLERUM_CONTEXT_MAPPER_URL` | Points to the host-context-controller endpoint used to resolve MCP servers for a context. | `CLERUM_CONTEXT_MAPPER_URL=http://host-context-controller-api-gateway.control-plane.svc.cluster.local:8081` |+| `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL` | Controls polling interval (milliseconds) for MCP server config changes from context mapper. | `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL=30000` |+| `HCC_AUTHORITY_MAX_STALENESS_MS` | Maximum time a previously published MCP fleet may survive an unavailable HCC (default 180000 ms; hard-capped at 600000 ms). It must be greater than `CLERUM_CONTEXT_MAPPER_POLL_INTERVAL` in cluster mode; invalid combinations fail startup. Minikube pins 60000 ms for its deterministic staleness E2E. | `HCC_AUTHORITY_MAX_STALENESS_MS=180000` |+| `CLERUM_HOST_CONFIG` | Provides a full Host spec as JSON in dev mode, overriding provider/model context env composition. | `CLERUM_HOST_CONFIG={"host":"dev-host","contextRef":"dev-context","secretRef":"dev-secret","model":{"provider":"openai","name":"gpt-5.4-mini"}}` |+| `CLERUM_MODEL_PROVIDER` | Chooses the model provider in dev mode when `CLERUM_HOST_CONFIG` is not set. | `CLERUM_MODEL_PROVIDER=openai` |+| `CLERUM_MODEL_NAME` | Overrides the default provider model name in dev mode. Used only when `CLERUM_MODEL_PROVIDER` is also set; an auto-detected provider keeps its default model. | `CLERUM_MODEL_NAME=gpt-5.4-mini` |+| `CLERUM_MCP_SERVERS` | Supplies dev MCP server definitions as JSON so you can run without context mapper. Each entry needs `name`, `contextRef`, `transport`, `enabled`, and a `status` object — a server whose `status.ready` is not `true` is parsed but skipped as "not ready". | `CLERUM_MCP_SERVERS=[{"name":"filesystem","contextRef":"dev-context","transport":{"type":"streamableHttp","url":"http://localhost:3101"},"enabled":true,"status":{"deployed":true,"ready":true}}]` |+| `CLERUM_MCP_TOOL_TIMEOUT_MS` | Per-call timeout passed to MCP SDK `Client.callTool(...)` requests. The effective timeout is still bounded by any caller workflow or snippet step budget. | `CLERUM_MCP_TOOL_TIMEOUT_MS=1500000` |+| `CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS` | Maximum total MCP SDK tool-call budget used at the SDK request boundary. Operators may tune it up to the existing workflow step ceiling; invalid values fail closed at runtime. | `CLERUM_MCP_TOOL_MAX_TOTAL_TIMEOUT_MS=1500000` |+| `OPENAI_API_KEY` | OpenAI API key used by dev mode when provider selection resolves to OpenAI. | `OPENAI_API_KEY=sk-xxxx` |+| `CLAUDE_API_KEY` | Anthropic API key used by dev mode when provider selection resolves to Claude. | `CLAUDE_API_KEY=sk-ant-xxxx` |+| `ZAI_API_KEY` | ZAI API key used by dev mode when provider selection resolves to ZAI. | `ZAI_API_KEY=zai-xxxx` |+| `BAILIAN_API_KEY` | Bailian API key used by dev mode when provider selection resolves to Bailian/Qwen. | `BAILIAN_API_KEY=ba-xxxx` |+| `CLERUM_AGENT_TASK_DELAY` | Minimum interval in milliseconds between task dispatches; does not delay individual tool iterations. | `CLERUM_AGENT_TASK_DELAY=3` |+| `CLERUM_AGENT_MAX_TASK_DURATION` | Maximum active task execution time in milliseconds (24 hours), accumulated across approval resumptions. Approval waiting is excluded. | `CLERUM_AGENT_MAX_TASK_DURATION=86400000` |+| `CLERUM_AGENT_MAX_TOOL_CALLS` | Maximum LLM/tool iterations per task; each iteration may call multiple tools. Configured per Host process. | `CLERUM_AGENT_MAX_TOOL_CALLS=1000` |+| `CLERUM_AGENT_MAX_QUEUE_SIZE` | Maximum pending queue size before new tasks are rejected/back-pressured. | `CLERUM_AGENT_MAX_QUEUE_SIZE=100` |+| `CLERUM_ENABLE_APPROVAL` | Enables the approval gate workflow for tool execution decisions. | `CLERUM_ENABLE_APPROVAL=true` |+| `CLERUM_APPROVAL_CONFIG` | JSON config for approval policy/channels when not sourced from Host CRD. | `CLERUM_APPROVAL_CONFIG={"defaultPolicy":"designated_approvers","channels":{"telegram":{"enabled":true,"approvers":["123456"]}}}` |+| `CLERUM_ENABLE_NUDGE` | Enables the nudge controller that pushes model/tool retry guidance when output is rejected. | `CLERUM_ENABLE_NUDGE=false` |+| `CLERUM_NUDGE_MAX_ITERATIONS` | Maximum number of nudge cycles allowed for a task. | `CLERUM_NUDGE_MAX_ITERATIONS=3` |+| `CLERUM_CONTEXT_MAX_TOKENS` | Token budget used by context compaction when the model's allowlist row carries no context window. `codex-subscription` and `grok-subscription` models use 256000 instead; the Host logs `context_window_resolved` with the window and its source. | `CLERUM_CONTEXT_MAX_TOKENS=100000` |+| `CLERUM_COMPACTION_PRE_PRUNE` | Master switch for pre-pruning the history (deduplication, one-lining, JSON truncation, media stripping). It runs only once the history reaches the compaction threshold (80% of the context window), before any compaction tier is chosen, and at history rehydration under the same threshold. On by default; set to `false` to disable every pass at once. | `CLERUM_COMPACTION_PRE_PRUNE=true` |+| `CLERUM_WORKSPACE_PATH` | Base workspace path used by native tools and workspace-aware operations. | `CLERUM_WORKSPACE_PATH=/workspace` |+| `CLERUM_SHELL_TIMEOUT` | Maximum execution time in milliseconds for shell native-tool commands. | `CLERUM_SHELL_TIMEOUT=1500000` |+| `CLERUM_TOOL_TIMEOUT` | Maximum execution time in milliseconds for native tool wrapper calls. This is separate from the MCP SDK request timeout controlled by `CLERUM_MCP_TOOL_TIMEOUT_MS`. | `CLERUM_TOOL_TIMEOUT=1500000` |+| `CLERUM_HTTP_ALLOWLIST` | Comma-separated allowlist of outbound hosts/URLs allowed for HTTP native tools. With an allowlist set, you can also skip per-call approval by adding `tools: { http_request: false }` under `Host.spec.approval` (see Host CRD). | `CLERUM_HTTP_ALLOWLIST=api.github.com,example.com` |+| `CLERUM_ENV_ALLOWLIST` | Comma-separated list of environment variables forwarded into native tool execution contexts. | `CLERUM_ENV_ALLOWLIST=PATH,HOME,USER,SHELL,LANG,TERM` |+| `CLERUM_MEMORY_MAX_SIZE` | Maximum bytes reserved for in-memory native tool buffers/state. | `CLERUM_MEMORY_MAX_SIZE=1048576` |+| `CLERUM_ENABLE_RESPONSE_ATTACHMENTS` | Enables attachment passthrough in runtime responses when tools return supported media. | `CLERUM_ENABLE_RESPONSE_ATTACHMENTS=true` |+| `CLERUM_ATTACHMENT_MAX_COUNT` | Maximum number of attachments returned per response. | `CLERUM_ATTACHMENT_MAX_COUNT=3` |+| `CLERUM_ATTACHMENT_MAX_BYTES` | Maximum decoded size per attachment in bytes (oversized payloads are dropped). | `CLERUM_ATTACHMENT_MAX_BYTES=52428800` |+| `CLERUM_ATTACHMENT_FILE_MAX_BYTES` | Maximum decoded bytes per incoming `kind:'file'` attachment, default and ceiling 11 MiB (11,534,336 bytes). A larger file rejects the message with `FILE_ATTACHMENT_TOO_LARGE`. File base64 has its own 16 MiB quota; the non-credited message share remains 6 MiB and the total body remains 24 MiB. A lower file limit does not change those body quotas. An invalid or above-ceiling setting stops the Host at startup. | `CLERUM_ATTACHMENT_FILE_MAX_BYTES=11534336` |+| `CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES` | Maximum bytes per native attachment text page, default 64 KiB (65,536 bytes), configurable up to 1 MiB (1,048,576 bytes). Truncated text continues from its byte offset. Native attachment pages remain inline and exempt from spillover. Byte size is independent of token cost; do not infer a fixed bytes-per-token ratio. An invalid or above-ceiling setting stops the Host at startup. | `CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES=65536` |+| `CLERUM_MEMORY_ENABLED` | Enables workspace memory persistence features. | `CLERUM_MEMORY_ENABLED=true` |+| `CLERUM_MEMORY_WORKSPACE_PATH` | Sets workspace directory path used when memory mode is enabled. | `CLERUM_MEMORY_WORKSPACE_PATH=/workspace` |+| `CLERUM_PERSONALIZATION_ENABLED` | Enables identity/personalization seeding behavior. | `CLERUM_PERSONALIZATION_ENABLED=true` |+| `CLERUM_IDENTITY_SEED` | JSON identity seed used to initialize personalization artifacts when enabled. | `CLERUM_IDENTITY_SEED={"displayName":"Clerum Agent","tone":"concise"}` |+| `NODE_ENV` | Influences production checks and defaults (for example strict host-name requirement in production). | `NODE_ENV=production` |### Host CRDdiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 660e6c099..a7fba88cd 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.480",+ "version": "0.1.481","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.480",+ "version": "0.1.481","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 8efd07c91..0724ea0e8 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.480",+ "version": "0.1.481","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/config.fileIngress.test.ts b/mcp-host/src/config.fileIngress.test.tsindex 8740b93a4..4bca85c7c 100644--- a/mcp-host/src/config.fileIngress.test.ts+++ b/mcp-host/src/config.fileIngress.test.ts@@ -45,18 +45,51 @@ describe('#666 file ingress configuration', () => {expect(config.attachmentFileMaxBytes).toBe(11_534_336)})- it('defaults the attachment text page to 64 KiB, under a quarter of the default compaction budget', async () => {+ it('defaults the attachment text page to 64 KiB without assuming a token-to-byte ratio', async () => {vi.resetModules()// Precondition: the default is what is under test, so the variable must be unset.expect(process.env.CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES).toBeUndefined()const { config } = await import('./config')expect(config.attachmentTextReadMaxBytes).toBe(65_536)expect(config.nativeTool.attachmentTextReadMaxBytes).toBe(65_536)- // The page is inline, so it is measured against the context budget, not- // the spillover threshold: bytes / 4 heuristic tokens, 0.8 pressure.- expect(Math.ceil(65_536 / 4)).toBeLessThanOrEqual(0.8 * config.contextMaxTokens * 0.25)})+ it.each([+ ['CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES', '1048577', '1048576'],+ ['CLERUM_ATTACHMENT_FILE_MAX_BYTES', '11534337', '11534336'],+ ])(+ 'rejects %s above its transport ceiling and accepts the ceiling',+ async (key, above, ceiling) => {+ vi.resetModules()+ vi.stubEnv(key, above)+ await expect(import('./config').then(() => undefined)).rejects.toThrow(key)++ vi.resetModules()+ vi.stubEnv(key, ceiling)+ const { config } = await import('./config')+ const field =+ key === 'CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES'+ ? config.attachmentTextReadMaxBytes+ : config.attachmentFileMaxBytes+ expect(field).toBe(Number(ceiling))+ }+ )++ it.each(['0', 'abc', '1.5', '10k'])(+ 'rejects an attachment text page of %j while accepting a smaller bounded page',+ async value => {+ vi.resetModules()+ vi.stubEnv('CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES', value)+ await expect(import('./config')).rejects.toThrow('CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES')++ vi.resetModules()+ vi.stubEnv('CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES', '4096')+ const { config } = await import('./config')+ expect(config.attachmentTextReadMaxBytes).toBe(4096)+ expect(config.nativeTool.attachmentTextReadMaxBytes).toBe(4096)+ }+ )+it.each(['0', 'abc', '2147483648'])('rejects a per-file attachment limit of %j at config load, naming the variable',async value => {diff --git a/mcp-host/src/config.ts b/mcp-host/src/config.tsindex a03284b51..612f74c88 100644--- a/mcp-host/src/config.ts+++ b/mcp-host/src/config.ts@@ -11,6 +11,7 @@ import {parseCodexToolPresentation,} from './core/orchestration/toolPresentationPolicy'import { ALL_PROVIDERS, type LlmProvider, descriptorFor, isLlmProvider } from './llm/registryCore'+import { logger } from './logger'import type { McpCatalogBootstrapConfig } from './mcp/grantProbe'import { HostSpec, McpServerInfo, MemoryConfig, ModelConfig, PersonalizationConfig } from './types'@@ -332,7 +333,12 @@ function getEnv(key: string, defaultValue?: string): string | undefined {return process.env[key] ?? defaultValue}-function getExecutionLimit(key: string, defaultValue: number, allowZero = false): number {+function getExecutionLimit(+ key: string,+ defaultValue: number,+ allowZero = false,+ maximum = 2_147_483_647+): number {const raw = getEnv(key)if (raw === undefined) return defaultValueconst value = Number(raw)@@ -340,7 +346,7 @@ function getExecutionLimit(key: string, defaultValue: number, allowZero = false)!/^\d+$/.test(raw) ||!Number.isSafeInteger(value) ||value < (allowZero ? 0 : 1) ||- value > 2_147_483_647+ value > maximum) {throw new Error(`${key} must be a valid bounded integer`)}@@ -361,8 +367,9 @@ function getEnvNumber(key: string, defaultValue: number): number {// Warn only when the value was actually set but invalid, so a typo'd env var// is visible in logs rather than silently ignored.if (!Number.isFinite(parsed) || parsed <= 0) {- console.warn(- `[config] ${key}=${value} is not a valid positive integer; using default ${defaultValue}`+ logger.warn(+ { component: 'Config', configurationKey: key, defaultValue },+ 'Invalid positive integer configuration; using its default')return defaultValue}@@ -452,17 +459,13 @@ function parseDevHostConfig(): HostSpec | undefined {try {const parsed = JSON.parse(configJson) as HostSpec- console.log('[Config] Parsed dev host config from CLERUM_HOST_CONFIG:')- console.log('[Config] host:', parsed.host)- console.log('[Config] contextRef:', parsed.contextRef)- console.log('[Config] secretRef:', parsed.secretRef)- console.log(- '[Config] model:',- parsed.model ? `${parsed.model.provider}/${parsed.model.name}` : 'not set'+ logger.info(+ { component: 'Config', modelConfigured: !!parsed.model },+ 'Parsed dev Host configuration')return parsed} catch (error) {- console.error('[Config] Failed to parse CLERUM_HOST_CONFIG:', error)+ logger.error({ component: 'Config', err: error }, 'Failed to parse dev Host configuration')return undefined}}@@ -487,10 +490,9 @@ function buildDevHostConfig(provider?: LlmProvider, modelName?: string): HostSpe}: undefined- console.log('[Config] Built dev host config from env vars:')- console.log(- '[Config] model:',- model ? `${model.provider}/${model.name}` : 'will auto-detect from API keys'+ logger.info(+ { component: 'Config', modelConfigured: !!model },+ 'Built dev Host configuration from environment')return {@@ -504,12 +506,13 @@ function buildDevHostConfig(provider?: LlmProvider, modelName?: string): HostSpeconst devMode = getEnvBool('CLERUM_DEV_MODE', false)// Read once: the top-level field documents the limit, `nativeTool` carries it// to `clerum__attachment_read` (#666). A page is shipped inline (the tool is-// spillover-exempt), so the default is sized against the context budget: at-// the tokenizer heuristic of 4 bytes per token, 65_536 bytes is ~16k tokens,-// about a fifth of the default compaction budget (0.8 * contextMaxTokens).+// spillover-exempt). Its byte ceiling is separate from the measured page and+// turn token budgets: dense data and output wrapping invalidate a fixed ratio.const attachmentTextReadMaxBytes = getExecutionLimit('CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES',- 65_536+ 65_536,+ false,+ 1_048_576)const configuredWorkflowEnabled = getEnvBool('CLERUM_WORKFLOW_ENABLED', false)const configuredRuntimeKind = resolveMcpHostRuntimeKind({@@ -642,13 +645,10 @@ function parseDevMcpServers(): McpServerInfo[] | undefined {try {const parsed = JSON.parse(serversJson) as McpServerInfo[]- console.log(`[Config] Parsed ${parsed.length} dev MCP server(s) from CLERUM_MCP_SERVERS`)- for (const server of parsed) {- console.log(`[Config] - ${server.name}: ${server.transport.url}`)- }+ logger.info({ component: 'Config', serverCount: parsed.length }, 'Parsed dev MCP servers')return parsed} catch (error) {- console.error('[Config] Failed to parse CLERUM_MCP_SERVERS:', error)+ logger.error({ component: 'Config', err: error }, 'Failed to parse dev MCP servers')return undefined}}@@ -663,12 +663,13 @@ function parseGuardrailsConfig(): GuardrailsConfig | undefined {if (!configJson) return undefinedtry {const parsed = JSON.parse(configJson) as GuardrailsConfig- console.log('[Config] Parsed guardrails config from CLERUM_GUARDRAILS_CONFIG:', {- rules: parsed.rules?.length ?? 0,- })+ logger.info(+ { component: 'Config', ruleCount: parsed.rules?.length ?? 0 },+ 'Parsed guardrails configuration'+ )return parsed} catch (error) {- console.error('[Config] Failed to parse CLERUM_GUARDRAILS_CONFIG:', error)+ logger.error({ component: 'Config', err: error }, 'Failed to parse guardrails configuration')return undefined}}@@ -681,12 +682,13 @@ function parseApprovalConfig(): ApprovalConfig | undefined {try {const parsed = JSON.parse(configJson) as ApprovalConfig- console.log('[Config] Parsed approval config from CLERUM_APPROVAL_CONFIG:')- console.log('[Config] defaultPolicy:', parsed.defaultPolicy)- console.log('[Config] channels:', Object.keys(parsed.channels || {}))+ logger.info(+ { component: 'Config', channelCount: Object.keys(parsed.channels || {}).length },+ 'Parsed approval configuration'+ )return parsed} catch (error) {- console.error('[Config] Failed to parse CLERUM_APPROVAL_CONFIG:', error)+ logger.error({ component: 'Config', err: error }, 'Failed to parse approval configuration')return undefined}}@@ -962,8 +964,9 @@ export const config: Config = {if (raw === 'memory' || raw === 'sqlite' || raw === 'dual') {return raw as 'memory' | 'sqlite' | 'dual'}- console.warn(- `[Config] CLERUM_SESSION_STORE='${raw}' is not recognized — falling back to 'memory'`+ logger.warn(+ { component: 'Config', configurationKey: 'CLERUM_SESSION_STORE', selectedMode: 'memory' },+ 'Unknown session store mode; using memory')return 'memory' as const})(),@@ -1042,7 +1045,12 @@ export const config: Config = {enableResponseAttachments: getEnvBool('CLERUM_ENABLE_RESPONSE_ATTACHMENTS', true),attachmentMaxCount: parseInt(getEnv('CLERUM_ATTACHMENT_MAX_COUNT', '3')!, 10),attachmentMaxBytes: parseInt(getEnv('CLERUM_ATTACHMENT_MAX_BYTES', '52428800')!, 10),- attachmentFileMaxBytes: getExecutionLimit('CLERUM_ATTACHMENT_FILE_MAX_BYTES', 11_534_336),+ attachmentFileMaxBytes: getExecutionLimit(+ 'CLERUM_ATTACHMENT_FILE_MAX_BYTES',+ 11_534_336,+ false,+ 11_534_336+ ),attachmentTextReadMaxBytes,fileReferenceMaxCount: FILE_REFERENCE_MAX_COUNT,activityBufferSize: parseInt(getEnv('MCP_HOST_ACTIVITY_BUFFER_SIZE', '1000')!, 10),@@ -1138,7 +1146,7 @@ export const config: Config = {try {seed = JSON.parse(seedJson) as Omit<PersonalizationConfig, 'enabled'>} catch {- console.error('[Config] Failed to parse CLERUM_IDENTITY_SEED')+ logger.error({ component: 'Config' }, 'Failed to parse identity seed configuration')}}return { enabled, ...seed }diff --git a/mcp-host/src/core/attachments/attachmentReadBudget.ts b/mcp-host/src/core/attachments/attachmentReadBudget.tsnew file mode 100644index 000000000..ab8216ea0--- /dev/null+++ b/mcp-host/src/core/attachments/attachmentReadBudget.ts@@ -0,0 +1,75 @@+export const MAX_ATTACHMENT_READS_PER_TURN = 32++export function attachmentReadBudgets(windowTokens: number): {+ pageTokens: number+ turnTokens: number+} {+ if (!Number.isSafeInteger(windowTokens) || windowTokens <= 0) {+ throw new Error('attachmentReadContextWindowTokens must be a positive safe integer')+ }+ return { pageTokens: Math.floor(windowTokens * 0.1), turnTokens: Math.floor(windowTokens * 0.3) }+}++export interface AttachmentReadLedgerSnapshot {+ reads: number+ spentTokens: number+ bytesRead: number+}++function boundedCount(value: number): boolean {+ return Number.isSafeInteger(value) && value >= 0+}++/** Owned by one turn; registry memoization does not determine its lifetime. */+export class AttachmentReadLedger {+ private state: AttachmentReadLedgerSnapshot = { reads: 0, spentTokens: 0, bytesRead: 0 }++ snapshot(): AttachmentReadLedgerSnapshot {+ return { ...this.state }+ }++ reset(): void {+ this.state = { reads: 0, spentTokens: 0, bytesRead: 0 }+ }++ restore(snapshot: AttachmentReadLedgerSnapshot): void {+ if (+ !Object.values(snapshot).every(boundedCount) ||+ snapshot.reads > MAX_ATTACHMENT_READS_PER_TURN+ ) {+ throw new Error('Invalid attachment read ledger snapshot')+ }+ this.state = { ...snapshot }+ }++ beginRead(): boolean {+ if (this.state.reads >= MAX_ATTACHMENT_READS_PER_TURN) return false+ this.state.reads += 1+ return true+ }++ pageAllowance(windowTokens: number, noticeCost: number): number {+ if (!boundedCount(noticeCost)) throw new Error('Invalid attachment notice cost')+ const limits = attachmentReadBudgets(windowTokens)+ // Leave one notice for every remaining call and the first exhausted call.+ const reserve = (MAX_ATTACHMENT_READS_PER_TURN - this.state.reads + 1) * noticeCost+ return Math.max(+ 0,+ Math.min(limits.pageTokens, limits.turnTokens - this.state.spentTokens - reserve)+ )+ }++ canEmit(windowTokens: number, cost: number): boolean {+ if (!boundedCount(cost)) return false+ const limits = attachmentReadBudgets(windowTokens)+ return cost <= limits.pageTokens && cost <= limits.turnTokens - this.state.spentTokens+ }++ debit(windowTokens: number, cost: number, bytesRead: number): void {+ if (!boundedCount(bytesRead) || !this.canEmit(windowTokens, cost)) {+ throw new Error('Attachment output cannot fit the remaining page/turn budget')+ }+ this.state.spentTokens += cost+ this.state.bytesRead += bytesRead+ }+}diff --git a/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts b/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.tsnew file mode 100644index 000000000..fa62b46c5--- /dev/null+++ b/mcp-host/src/core/extensions/__tests__/prePrune.attachmentPageCollapse.test.ts@@ -0,0 +1,247 @@+/**+ * C17 — `collapseEarlierAttachmentPages` pre-prune pass.+ *+ * Covers the frozen contract: only genuine native text-page results of+ * `clerum__attachment_read` that live strictly before the LATEST user message+ * collapse into small stubs. Current-turn pages, foreign tools, malformed+ * shapes, linkage fields and the protected-tail semantics of the other passes+ * stay untouched. The pass is pressure-gated inside `prePrune`.+ */+import { describe, expect, it } from 'vitest'+import { validateToolLinkages } from '../../orchestration/toolUseLoop'+import type { ChatMessage } from '../../types'+import {+ DEFAULT_PRE_PRUNE_OPTIONS,+ type PrePruneOptions,+ type PrePrunePressure,+ prePrune,+} from '../prePrune'+import * as prePruneModule from '../prePrune'++const PAGE_TEXT = 'confidential page body '.repeat(200)+const FILENAME = 'strategy-2027.md'++function userMsg(content: string): ChatMessage {+ return { role: 'user', content }+}++function assistantToolCall(id: string, name: string, args: Record<string, unknown>): ChatMessage {+ return { role: 'assistant', content: '', tool_calls: [{ id, name, arguments: args }] }+}++function attachmentToolResult(id: string, content: string): ChatMessage {+ return { role: 'tool', tool_call_id: id, name: 'clerum__attachment_read', content }+}++function nativePage(overrides: Record<string, unknown> = {}): string {+ return JSON.stringify({+ attachmentId: 'att_1',+ referenceId: 'ref_sha256 deadbeef',+ kind: 'text',+ byteRange: { offset: 0, length: 65536 },+ truncated: true,+ text: PAGE_TEXT,+ ...overrides,+ })+}++function wrapped(content: string, toolName = 'clerum__attachment_read'): string {+ return `<tool_output name="${toolName}" sanitized="true">\n${content}\n</tool_output>`+}++function stubMarker(): string {+ return '[earlier attachment page collapsed; re-read with clerum__attachment_read if needed]'+}++const PRESSURE_ON: PrePrunePressure = { inputTokens: 900, contextWindowTokens: 1000 }+const PRESSURE_OFF: PrePrunePressure = { inputTokens: 799, contextWindowTokens: 1000 }+const OPTIONS: PrePruneOptions = { ...DEFAULT_PRE_PRUNE_OPTIONS }++/** Two-turn conversation: one old page, one current-turn page. */+function conversation(): ChatMessage[] {+ return [+ userMsg(`first turn, see ${FILENAME}`),+ assistantToolCall('tc_old', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_old', nativePage()),+ userMsg('second turn'),+ assistantToolCall('tc_cur', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_cur', nativePage({ byteRange: { offset: 65536, length: 100 } })),+ ]+}++describe('C17 collapseEarlierAttachmentPages — export', () => {+ it('exports the frozen collapseEarlierAttachmentPages function', () => {+ expect(typeof prePruneModule.collapseEarlierAttachmentPages).toBe('function')+ })+})++describe('C17 collapseEarlierAttachmentPages — direct pass', () => {+ it('collapses a genuine old-turn native page and preserves identity + paging + linkage', () => {+ const messages = conversation()+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)++ expect(out).not.toBe(messages)+ expect(out[2]).not.toBe(messages[2])+ const stub = JSON.parse(out[2].content) as Record<string, unknown>+ expect(stub.attachmentId).toBe('att_1')+ expect(stub.referenceId).toBe('ref_sha256 deadbeef')+ expect(stub.kind).toBe('text')+ expect(stub.byteRange).toEqual({ offset: 0, length: 65536 })+ expect(stub.truncated).toBe(true)+ expect(stub.text).toBe(stubMarker())+ expect(out[2].tool_call_id).toBe('tc_old')+ expect(out[2].name).toBe('clerum__attachment_read')+ expect(() => validateToolLinkages(out)).not.toThrow()+ })++ it('leaks no raw page text or filename into the stub', () => {+ const messages = conversation()+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)+ expect(out[2].content).not.toContain('confidential page body')+ expect(out[2].content).not.toContain(FILENAME)+ expect(out[2].content.length).toBeLessThan(messages[2].content.length / 4)+ })++ it('preserves the sanitized wrapper around the stub', () => {+ const messages = conversation()+ messages[2] = attachmentToolResult('tc_old', wrapped(nativePage()))+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)++ expect(+ out[2].content.startsWith('<tool_output name="clerum__attachment_read" sanitized="true">')+ ).toBe(true)+ expect(out[2].content.endsWith('</tool_output>')).toBe(true)+ const inner = out[2].content.split('\n')[1]!+ expect((JSON.parse(inner) as Record<string, unknown>).text).toBe(stubMarker())+ })++ it('keeps current-turn pages byte-identical and referentially intact', () => {+ const messages = conversation()+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)+ expect(out[5]).toBe(messages[5])+ expect(out[5].content).toBe(messages[5].content)+ })++ it('keeps untouched messages referentially identical and originals immutable', () => {+ const messages = conversation()+ const before = structuredClone(messages)+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)+ expect(messages).toEqual(before)+ for (const i of [0, 1, 3, 4, 5]) expect(out[i]).toBe(messages[i])+ })++ it('is idempotent: a second pass returns the same array reference', () => {+ const messages = conversation()+ const once = prePruneModule.collapseEarlierAttachmentPages(messages)+ const twice = prePruneModule.collapseEarlierAttachmentPages(once)+ expect(twice).toBe(once)+ })++ it('leaves other tools, malformed and foreign shapes unchanged', () => {+ const foreignTool: ChatMessage = {+ role: 'tool',+ tool_call_id: 'tc_gfs',+ name: 'clerum__gfs_read',+ content: nativePage(),+ }+ const malformed: ChatMessage[] = [+ userMsg('q'),+ assistantToolCall('tc_m', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_m', '{"attachmentId":"att_1","kind":"text"'),+ assistantToolCall('tc_x', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_x', nativePage({ truncated: 'yes' })),+ assistantToolCall('tc_b', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_b', nativePage({ kind: 'binary' })),+ assistantToolCall('tc_e', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_e', nativePage({ extra: true })),+ userMsg('now'),+ ]+ const wrappedForeign = wrapped(nativePage(), 'clerum__gfs_read')+ malformed.push(attachmentToolResult('tc_w', wrappedForeign))++ const input = [foreignTool, ...malformed]+ const out = prePruneModule.collapseEarlierAttachmentPages(input)+ expect(out).toBe(input)+ })++ it('uses the latest-user boundary, not the protected-tail boundary', () => {+ // Four user messages: protectedTailTurns=3 protects from user B onwards+ // (index 3), but the collapse contract protects only from the LATEST user+ // (index 7). Page p2 at index 5 is before the latest user and must collapse+ // even though it sits inside the default protected tail.+ const messages: ChatMessage[] = [+ userMsg('A'),+ assistantToolCall('tc_1', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_1', nativePage()),+ userMsg('B'),+ assistantToolCall('tc_2', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_2', nativePage()),+ userMsg('C'),+ userMsg('D'),+ assistantToolCall('tc_3', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_3', nativePage()),+ ]+ const out = prePruneModule.collapseEarlierAttachmentPages(messages)+ expect(JSON.parse(out[2].content).text).toBe(stubMarker())+ expect(JSON.parse(out[5].content).text).toBe(stubMarker())+ expect(out[9]).toBe(messages[9])+ })+})++describe('C17 prePrune — pressure-gated wiring', () => {+ it('runs the pass before dedup under qualifying pressure', () => {+ const messages: ChatMessage[] = [+ userMsg('A'),+ assistantToolCall('tc_1', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_1', nativePage()),+ assistantToolCall('tc_2', 'clerum__attachment_read', { attachmentId: 'att_1' }),+ attachmentToolResult('tc_2', nativePage()),+ userMsg('B'),+ userMsg('C'),+ userMsg('D'),+ ]+ const result = prePrune(messages, OPTIONS, PRESSURE_ON)+ expect(result.passesApplied[0]).toBe('attachment_page_collapse')+ expect(result.passesApplied).toContain('dedup')+ expect(JSON.parse(result.messages[2].content).text).toBe(stubMarker())+ expect(result.messages[4].content).toBe('[duplicate of tool_call_id=tc_1]')+ })++ it('does not collapse without pressure', () => {+ const messages = conversation()+ const result = prePrune(messages, OPTIONS)+ expect(result.messages[2]).toBe(messages[2])+ expect(result.passesApplied).not.toContain('attachment_page_collapse')+ })++ it('does not collapse below the 0.8 window threshold', () => {+ const messages = conversation()+ const result = prePrune(messages, OPTIONS, PRESSURE_OFF)+ expect(result.messages[2]).toBe(messages[2])+ })++ it('does not collapse when the option is explicitly disabled', () => {+ const messages = conversation()+ const result = prePrune(+ messages,+ { ...OPTIONS, attachmentPageCollapseEnabled: false },+ PRESSURE_ON+ )+ expect(result.messages[2]).toBe(messages[2])+ })++ it('rejects non-finite or non-positive windows and non-finite input tokens', () => {+ const messages = conversation()+ const bad: PrePrunePressure[] = [+ { inputTokens: 900, contextWindowTokens: 0 },+ { inputTokens: 900, contextWindowTokens: Number.POSITIVE_INFINITY },+ { inputTokens: 900, contextWindowTokens: Number.NaN },+ { inputTokens: Number.POSITIVE_INFINITY, contextWindowTokens: 1000 },+ { inputTokens: Number.NaN, contextWindowTokens: 1000 },+ ]+ for (const pressure of bad) {+ const result = prePrune(messages, OPTIONS, pressure)+ expect(result.messages[2]).toBe(messages[2])+ }+ })+})diff --git a/mcp-host/src/core/extensions/__tests__/toolMessageBudget.test.ts b/mcp-host/src/core/extensions/__tests__/toolMessageBudget.test.tsnew file mode 100644index 000000000..0b6c01e93--- /dev/null+++ b/mcp-host/src/core/extensions/__tests__/toolMessageBudget.test.ts@@ -0,0 +1,60 @@+import { describe, expect, it } from 'vitest'+import { get_encoding } from 'tiktoken'+import type { TokenCounter } from '../../tokenizer/tokenCounter'+import type { ChatMessage } from '../../types'+import * as contextManager from '../contextManager'++type Measure = (message: ChatMessage, counter: TokenCounter | undefined, dryRun: boolean) => number++function measure(): Measure {+ const candidate = (contextManager as unknown as { toolMessageBudgetTokens?: Measure })+ .toolMessageBudgetTokens+ expect(candidate).toBeTypeOf('function')+ return candidate!+}++describe('tool-message token budget', () => {+ it('accounts for the dense body and tool-message framing with a real BPE witness', () => {+ const encoder = get_encoding('cl100k_base')+ const content = Buffer.from('a deterministic public byte sequence')+ .toString('base64')+ .repeat(2048)+ const message: ChatMessage = {+ role: 'tool',+ name: 'clerum__attachment_read',+ tool_call_id: 'call_public',+ content,+ }+ const whole = encoder.encode(content, [], []).length+ const count = measure()(message, undefined, true)+ expect(count).toBeGreaterThanOrEqual(whole)+ expect(count).toBeGreaterThan(Math.ceil(Buffer.byteLength(content) / 4))+ expect(measure()({ ...message, content: '' }, undefined, true)).toBeGreaterThan(0)+ encoder.free()+ })++ it('uses the larger applicable provider estimate and does not call the network counter', () => {+ let networkCalls = 0+ const counter: TokenCounter = {+ providerName: 'unknown',+ modelName: 'public-unit-fixture',+ count: async () => {+ networkCalls += 1+ return 90_000+ },+ countSync: () => 20_000,+ warmup: async () => {},+ recordObservedUsage: () => {},+ lastObservedInputTokens: () => null,+ }+ const message: ChatMessage = {+ role: 'tool',+ name: 'bounded_tool',+ tool_call_id: 'call_1',+ content: 'small page',+ }+ expect(measure()(message, counter, false)).toBe(20_000)+ expect(measure()(message, counter, true)).toBeLessThan(20_000)+ expect(networkCalls).toBe(0)+ })+})diff --git a/mcp-host/src/core/extensions/contextManager.ts b/mcp-host/src/core/extensions/contextManager.tsindex fff24f0d8..bde1fe346 100644--- a/mcp-host/src/core/extensions/contextManager.ts+++ b/mcp-host/src/core/extensions/contextManager.ts@@ -25,7 +25,8 @@ import { parseStructuredSummary } from '../conversation/structuredSummaryParser'import { ContextManageOptions, ContextManager, LlmPort } from '../interfaces'import type { AgentEventEmitter } from '../interfaces'import { validateToolLinkages } from '../orchestration/toolUseLoop'-import { heuristicCountTools } from '../tokenizer/heuristic'+import { bpeTokenEstimate } from '../tokenizer/bpeEstimate'+import { heuristicCount, heuristicCountTools } from '../tokenizer/heuristic'import { tokenizerDryrunDelta, tokenizerDryrunTierMismatchTotal } from '../tokenizer/metrics'import type { TokenCounter } from '../tokenizer/tokenCounter'import { ChatMessage, CompactionState, Conversation, ToolDefinition } from '../types'@@ -168,6 +169,22 @@ export async function tierDecisionTokens(return tokenCounter.count(messages, tools)}+/** Measure the complete emitted tool message with the larger applicable estimate. */+export function toolMessageBudgetTokens(+ message: ChatMessage,+ tokenCounter: TokenCounter | undefined,+ dryRun: boolean+): number {+ const applicable =+ !tokenCounter || dryRun ? heuristicCount([message]) : tokenCounter.countSync([message])+ const bpe =+ bpeTokenEstimate(message.content ?? '') ++ bpeTokenEstimate(message.name ?? '') ++ bpeTokenEstimate(message.tool_call_id ?? '') ++ 6+ return Math.max(applicable, bpe)+}+/*** The messages a pressure count measures: the loop's messages, preceded by the* system prompt as the one system message it becomes on the wire (R9-14). The@@ -375,7 +392,10 @@ export class PressureContextManager implements ContextManager {// set without invoking the tier (no LLM call, no workspace write).let working = messagesif (this.prePruneEnabled) {- const result = prePrune(messages, this.prePruneOptions)+ const result = prePrune(messages, this.prePruneOptions, {+ inputTokens: pressure * this.maxTokens,+ contextWindowTokens: this.maxTokens,+ })working = result.messagesif (result.passesApplied.length > 0) {validateToolLinkages(working)diff --git a/mcp-host/src/core/extensions/prePrune.ts b/mcp-host/src/core/extensions/prePrune.tsindex 38b1ac8b9..93db8b091 100644--- a/mcp-host/src/core/extensions/prePrune.ts+++ b/mcp-host/src/core/extensions/prePrune.ts@@ -1,15 +1,17 @@/*** T1.2 — Pre-pruning pass executed BEFORE invoking the LLM.*- * Four deterministic, LLM-free passes that try to recover context tokens+ * Five deterministic, LLM-free passes that try to recover context tokens* before the tiered compactor (MoveToWorkspace / Summarize / Truncate)* pays an LLM call:*- * 1. Dedup tool outputs by canonical SHA-256 content hash.- * 2. One-line-summarize oversized tool outputs that live outside the+ * 1. Collapse earlier `clerum__attachment_read` text pages into small+ * identity+range stubs (pressure-gated by the caller).+ * 2. Dedup tool outputs by canonical SHA-256 content hash.+ * 3. One-line-summarize oversized tool outputs that live outside the* protected tail.- * 3. JSON-safe truncate oversized `assistant.tool_calls[].arguments`.- * 4. Strip historical media (image content parts) from anything before+ * 4. JSON-safe truncate oversized `assistant.tool_calls[].arguments`.+ * 5. Strip historical media (image content parts) from anything before* the latest user message.** Philosophy: local CPU is cheap compared to LLM tokens. Each pass is@@ -43,6 +45,16 @@ export const clerumPrePruneSavingsTokensTotal = new Counter({help: 'Heuristic tokens reclaimed by T1.2 pre-pruning (sum of pre - post per call).',})+/**+ * Authoritative pressure snapshot supplied by the context manager. The+ * attachment-page collapse only runs under real pressure; consumers that do+ * not pass this object never collapse pages.+ */+export interface PrePrunePressure {+ inputTokens: number+ contextWindowTokens: number+}+export interface PrePruneOptions {/** Cuántos turns proteger del tail (default 3). */protectedTailTurns: number@@ -71,6 +83,12 @@ export interface PrePruneOptions {oneLineSummariesEnabled: booleanjsonSafeTruncateEnabled: booleanstripMediaEnabled: boolean+ /**+ * C17 — collapse earlier `clerum__attachment_read` text pages under+ * pressure. Defaults to true; the pass still requires an explicit,+ * finite-positive `pressure` argument to fire.+ */+ attachmentPageCollapseEnabled?: boolean}export const DEFAULT_PRE_PRUNE_OPTIONS: PrePruneOptions = {@@ -81,9 +99,15 @@ export const DEFAULT_PRE_PRUNE_OPTIONS: PrePruneOptions = {oneLineSummariesEnabled: true,jsonSafeTruncateEnabled: true,stripMediaEnabled: true,+ attachmentPageCollapseEnabled: true,}const DUPLICATE_PREFIX = '[duplicate of tool_call_id='+const ATTACHMENT_READ_TOOL_NAME = 'clerum__attachment_read'+const ATTACHMENT_PAGE_COLLAPSE_MARKER =+ '[earlier attachment page collapsed; re-read with clerum__attachment_read if needed]'+const WRAPPED_ATTACHMENT_OUTPUT_PATTERN =+ /^<tool_output name="clerum__attachment_read" sanitized="(true|false)">\n([\s\S]*)\n<\/tool_output>$/export interface PrePruneResult {messages: ChatMessage[]@@ -100,13 +124,24 @@ export interface PrePruneResult {*/export function prePrune(messages: ChatMessage[],- opts: PrePruneOptions = DEFAULT_PRE_PRUNE_OPTIONS+ opts: PrePruneOptions = DEFAULT_PRE_PRUNE_OPTIONS,+ pressure?: PrePrunePressure): PrePruneResult {const preTokens = heuristicCount(messages)const protectedTailStart = computeProtectedTailStart(messages, opts.protectedTailTurns)const passesApplied: string[] = []let out = messages+ if (+ opts.attachmentPageCollapseEnabled !== false &&+ attachmentCollapsePressureQualifies(pressure)+ ) {+ const next = collapseEarlierAttachmentPages(out)+ if (next !== out) {+ passesApplied.push('attachment_page_collapse')+ out = next+ }+ }if (opts.dedupEnabled) {const next = dedupToolResults(out, protectedTailStart)if (next !== out) {@@ -140,6 +175,18 @@ export function prePrune(return { messages: out, preTokens, postTokens, passesApplied }}+function attachmentCollapsePressureQualifies(pressure: PrePrunePressure | undefined): boolean {+ if (!pressure) return false+ const { inputTokens, contextWindowTokens } = pressure+ return (+ Number.isFinite(inputTokens) &&+ inputTokens >= 0 &&+ Number.isFinite(contextWindowTokens) &&+ contextWindowTokens > 0 &&+ inputTokens >= 0.8 * contextWindowTokens+ )+}+/*** Compute the protected-tail start index. The last K user messages start* the protected region; pre-prune leaves `messages[protectedTailStart..]`@@ -164,6 +211,126 @@ export function computeProtectedTailStart(messages: ChatMessage[], tailTurns: nureturn 0}+// ─── Pass 0 — Collapse earlier attachment text pages ────────────────────────++/** The genuine native text-page result of `clerum__attachment_read`. */+interface AttachmentTextPage {+ attachmentId: string+ referenceId: string+ kind: 'text'+ byteRange: { offset: number; length: number }+ truncated: boolean+ text: string+}++/**+ * C17 — collapse the text of `clerum__attachment_read` pages that belong to+ * turns strictly before the LATEST user message. The stub keeps every field+ * the model needs to re-read the page on demand — attachment/reference+ * identity, byte range, truncation flag and the tool linkage — and replaces+ * only `text` with a fixed marker. Pages of the current turn (from the latest+ * user message onwards) stay byte-identical, as do foreign tools, malformed+ * payloads and binary results.+ *+ * Boundary note: this pass deliberately uses the latest-user boundary, not+ * `computeProtectedTailStart`. Attachment pages are the one payload the model+ * can re-fetch deterministically, so under pressure they collapse even inside+ * the protected tail, while every other pass keeps its protected-tail+ * semantics unchanged.+ */+export function collapseEarlierAttachmentPages(messages: ChatMessage[]): ChatMessage[] {+ const lastUserIndex = findLastUserIndex(messages)+ if (lastUserIndex <= 0) return messages+ let mutated: ChatMessage[] | null = null+ for (let i = 0; i < lastUserIndex; i++) {+ const msg = messages[i]+ if (msg.role !== 'tool' || msg.name !== ATTACHMENT_READ_TOOL_NAME) continue+ if (typeof msg.tool_call_id !== 'string' || msg.tool_call_id.length === 0) continue+ const parsed = parseNativeAttachmentTextPage(msg.content)+ if (!parsed) continue+ if (parsed.page.text === ATTACHMENT_PAGE_COLLAPSE_MARKER) continue+ if (mutated === null) mutated = messages.slice()+ mutated[i] = { ...msg, content: buildAttachmentPageStub(parsed) }+ }+ return mutated ?? messages+}++function findLastUserIndex(messages: ChatMessage[]): number {+ for (let i = messages.length - 1; i >= 0; i--) {+ if (messages[i].role === 'user') return i+ }+ return -1+}++function parseNativeAttachmentTextPage(+ content: string+): { page: AttachmentTextPage; wrapperSanitized: string | null } | null {+ let candidate = content+ let wrapperSanitized: string | null = null+ const wrapped = WRAPPED_ATTACHMENT_OUTPUT_PATTERN.exec(content)+ if (wrapped) {+ candidate = wrapped[2]!+ wrapperSanitized = wrapped[1]!+ }+ let parsed: unknown+ try {+ parsed = JSON.parse(candidate)+ } catch {+ return null+ }+ if (!isAttachmentTextPage(parsed)) return null+ return { page: parsed, wrapperSanitized }+}++function isAttachmentTextPage(value: unknown): value is AttachmentTextPage {+ if (!value || typeof value !== 'object' || Array.isArray(value)) return false+ const obj = value as Record<string, unknown>+ const keys = Object.keys(obj)+ if (keys.length !== 6) return false+ for (const key of ['attachmentId', 'byteRange', 'kind', 'referenceId', 'text', 'truncated']) {+ if (!(key in obj)) return false+ }+ if (typeof obj.attachmentId !== 'string' || obj.attachmentId.length === 0) return false+ if (typeof obj.referenceId !== 'string' || obj.referenceId.length === 0) return false+ if (obj.kind !== 'text') return false+ if (typeof obj.text !== 'string') return false+ if (typeof obj.truncated !== 'boolean') return false+ const range = obj.byteRange+ if (!range || typeof range !== 'object' || Array.isArray(range)) return false+ const rangeKeys = Object.keys(range)+ if (rangeKeys.length !== 2 || !('offset' in range) || !('length' in range)) return false+ const offset = (range as Record<string, unknown>).offset+ const length = (range as Record<string, unknown>).length+ if (+ typeof offset !== 'number' ||+ !Number.isSafeInteger(offset) ||+ offset < 0 ||+ typeof length !== 'number' ||+ !Number.isSafeInteger(length) ||+ length < 0+ ) {+ return false+ }+ return true+}++function buildAttachmentPageStub(parsed: {+ page: AttachmentTextPage+ wrapperSanitized: string | null+}): string {+ const { page, wrapperSanitized } = parsed+ const stub = JSON.stringify({+ attachmentId: page.attachmentId,+ referenceId: page.referenceId,+ kind: 'text',+ byteRange: { offset: page.byteRange.offset, length: page.byteRange.length },+ truncated: page.truncated,+ text: ATTACHMENT_PAGE_COLLAPSE_MARKER,+ })+ if (wrapperSanitized === null) return stub+ return `<tool_output name="${ATTACHMENT_READ_TOOL_NAME}" sanitized="${wrapperSanitized}">\n${stub}\n</tool_output>`+}+// ─── Pass 1 — Dedup tool outputs ────────────────────────────────────────────/**diff --git a/mcp-host/src/core/interfaces.ts b/mcp-host/src/core/interfaces.tsindex 2663e6d01..dd97a9c78 100644--- a/mcp-host/src/core/interfaces.ts+++ b/mcp-host/src/core/interfaces.ts@@ -84,6 +84,8 @@ export interface ExecutionContext {timeoutMs?: numbersignal?: AbortSignalvisualInput?: VisualInputContext+ /** Pure estimate of the complete sanitized/wrapped message for this call. */+ measureResult?: (content: string) => number/*** Tool calls this as output becomes available (not required to be line-aligned* — the ring buffer handles line boundaries).@@ -91,6 +93,11 @@ export interface ExecutionContext {onOutput(chunk: string): void}+export interface ToolEmissionContext {+ measureContent(content: string): number+ renderContent(content: string): string+}+export interface Tool {name(): stringdescription(): string@@ -123,6 +130,11 @@ export interface Tool {* configured threshold.*/spilloverExempt?(): boolean+ /** Final budget/revocation fence, after result transforms and before publication. */+ finalizeResult?(+ result: ToolResult,+ context: ToolEmissionContext+ ): ToolResult | Promise<ToolResult>}// ─── Channel ────────────────────────────────────────────────@@ -140,6 +152,8 @@ export interface Safety {validateToolParams(toolName: string, params: Record<string, unknown>): ValidationResultsanitizeOutput(toolName: string, output: string): SanitizedOutputwrapForLlm(toolName: string, content: string, wasSanitized: boolean): string+ /** Same sanitizer and wrapper, without per-call observability effects. */+ previewOutputForLlm?(toolName: string, content: string): string}// ─── Storage ────────────────────────────────────────────────@@ -171,6 +185,8 @@ export interface PromptBuilder {export interface ToolOutputProcessor {beforeExecution(toolName: string, params: Record<string, unknown>): ValidationResultafterExecution(toolName: string, output: ToolOutput): string+ /** Pure preparation for bounded native tools; never executes a tool or hook. */+ previewForLlm?(toolName: string, content: string): string}export interface LoopController {diff --git a/mcp-host/src/core/orchestration/loopConfig.ts b/mcp-host/src/core/orchestration/loopConfig.tsindex 7a88ebb8a..ab8a1ba0e 100644--- a/mcp-host/src/core/orchestration/loopConfig.ts+++ b/mcp-host/src/core/orchestration/loopConfig.ts@@ -43,6 +43,8 @@ export interface LoopConfig {loopController: LoopControllercontextManager: ContextManagertoolOutputProcessor: ToolOutputProcessor+ /** Synchronous estimate of the final model-visible tool message. Never network. */+ measureToolMessage?: (message: ChatMessage) => number/*** The system prompt `reasoning` sends with a request that presents `tools`,diff --git a/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts b/mcp-host/src/core/orchestration/toolUseLoopSingleTool.tsindex 76d80a7b3..c29f36027 100644--- a/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts+++ b/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts@@ -28,8 +28,10 @@ export async function executeSingleTool(| 'taskId'| 'visualInput'| 'abortSignal'+ | 'measureToolMessage'>,- iteration?: number+ iteration?: number,+ transformResult?: (result: ToolResult) => Promise<ToolResult>): Promise<ToolResult> {const { toolRegistry, toolOutputProcessor, events, toolTimeout } = config@@ -62,6 +64,24 @@ export async function executeSingleTool(}}+ const renderContent = (content: string): string => {+ if (!tool.requiresSanitization()) return content+ if (!toolOutputProcessor.previewForLlm) {+ throw new Error('measureResult requires ToolOutputProcessor.previewForLlm')+ }+ return toolOutputProcessor.previewForLlm(call.name, content)+ }+ const measureContent = (content: string): number => {+ if (!config.measureToolMessage)+ throw new Error('measureToolMessage is required for bounded native output')+ return config.measureToolMessage({+ role: 'tool',+ name: call.name,+ tool_call_id: call.id,+ content,+ })+ }+events.emit({type: 'tool:called',data: { toolName: call.name, toolCallId: call.id },@@ -72,11 +92,15 @@ export async function executeSingleTool(const executionContext: ExecutionContext = {onOutput: chunk => ringBuffer?.append(chunk),visualInput: config.visualInput,+ measureResult: config.measureToolMessage+ ? content => measureContent(renderContent(content))+ : undefined,}let watcherId: NodeJS.Timeout | null = nullconst watcherStartedAt = Date.now()let wantsWatcher = false+ let finalizingResult = falsetry {wantsWatcher =typeof tool.supportsProgressOutput === 'function' &&@@ -216,7 +240,7 @@ export async function executeSingleTool(}}- return {+ let result: ToolResult = {tool_call_id: call.id,name: call.name,content: finalContent,@@ -226,7 +250,25 @@ export async function executeSingleTool(rawContent: output.content,spillover_ref: spilloverRef,}+ // Post-result hooks execute once. Measure after them; a preview never+ // repeats a hook, tool execution, progress event or spillover write.+ finalizingResult = true+ if (transformResult) result = await transformResult(result)+ if (config.measureToolMessage) {+ result = { ...result, emittedMessageCost: measureContent(result.content) }+ }+ if (tool.finalizeResult) {+ result = await tool.finalizeResult(result, { measureContent, renderContent })+ const finalCost = measureContent(result.content)+ if (result.emittedMessageCost !== finalCost) {+ throw new Error('Final tool-message measurement does not match its budget debit')+ }+ }+ return result} catch (err) {+ // A result-policy/finalization failure must stop publication. Turning it+ // into an ordinary tool error could make the model repeat completed work.+ if (finalizingResult) throw errconst errorMessage =err instanceof ToolError ? err.message : `Tool execution failed: ${(err as Error).message}`diff --git a/mcp-host/src/core/orchestration/toolUseLoopToolBatch.ts b/mcp-host/src/core/orchestration/toolUseLoopToolBatch.tsindex 2d143948e..bde7551cf 100644--- a/mcp-host/src/core/orchestration/toolUseLoopToolBatch.ts+++ b/mcp-host/src/core/orchestration/toolUseLoopToolBatch.ts@@ -407,22 +407,17 @@ export async function executeToolCalls(}const progressStart = reportToolStart(config, call, iteration, i, calls.length, llmTextContent)- let toolResult = await executeSingleTool(call, config, iteration)-- // PostToolUse redaction (spec §6.2 / §10 #3): installed `post_tool_use` hooks- // may redact the model-visible result `content` (never `is_error`). Only the- // LLM-message `content` is touched — `rawContent` (UI preview) is left intact.- // Applied here — before the abort push, the U5 connect-required suspend, and- // the main push below — so every downstream consumer sees the redacted result.- if (config.guardrails?.transformResult && toolIdentity) {- const view = await config.guardrails.transformResult(toolIdentity, call.arguments, {- content: toolResult.content,- isError: toolResult.is_error,- })- if (view.content !== toolResult.content) {- toolResult = { ...toolResult, content: view.content }- }- }+ const transformResult =+ config.guardrails?.transformResult && toolIdentity+ ? async (result: ToolResult): Promise<ToolResult> => {+ const view = await config.guardrails!.transformResult!(toolIdentity, call.arguments, {+ content: result.content,+ isError: result.is_error,+ })+ return view.content === result.content ? result : { ...result, content: view.content }+ }+ : undefined+ const toolResult = await executeSingleTool(call, config, iteration, transformResult)// Retain policy-processed output before a subsequent tool can throw.if (config.onAttachments && toolResult.attachments?.length) {diff --git a/mcp-host/src/core/safety/safety.ts b/mcp-host/src/core/safety/safety.tsindex 8870dee24..0206c7567 100644--- a/mcp-host/src/core/safety/safety.ts+++ b/mcp-host/src/core/safety/safety.ts@@ -1,3 +1,4 @@+import { logger } from '../../logger'import { Safety } from '../interfaces'import { isPrivateIp } from '../tools/httpRequest'import { SanitizedOutput, ValidationResult } from '../types'@@ -22,6 +23,11 @@ import { SanitizedOutput, ValidationResult } from '../types'*/export type SecretEntriesProvider = () => Array<{ name: string; value: string }>+function wrapToolOutput(toolName: string, content: string, wasSanitized: boolean): string {+ const escaped = content.replace(/<\/tool_output>/gi, '</tool_output>')+ return `<tool_output name="${toolName}" sanitized="${wasSanitized}">\n${escaped}\n</tool_output>`+}+export class BasicSafety implements Safety {/*** Optional callback that returns ConfigStore-managed secret values. When@@ -139,7 +145,10 @@ export class BasicSafety implements Safety {}const valid = errors.length === 0- console.log(`[NewCore:Safety] validateInput → length=${input?.length ?? 0}, passed=${valid}`)+ logger.info(+ { component: 'Safety', length: input?.length ?? 0, passed: valid },+ 'Input validated'+ )return { is_valid: valid, errors }}@@ -228,8 +237,14 @@ export class BasicSafety implements Safety {const result = this.sanitizeFreeformContent(output, {secretWarning: `Potential secret detected in ${toolName} output`,})- console.log(- `[NewCore:Safety] sanitizeOutput → tool=${toolName}, sanitized=${result.was_modified}, warnings=${result.warnings.length}`+ logger.info(+ {+ component: 'Safety',+ toolName,+ sanitized: result.was_modified,+ warningCount: result.warnings.length,+ },+ 'Tool output sanitized')return result}@@ -239,8 +254,9 @@ export class BasicSafety implements Safety {secretWarning: 'Potential secret detected in assistant response',extraFilters: BasicSafety.ASSISTANT_RESPONSE_FILTER_PATTERNS,})- console.log(- `[NewCore:Safety] sanitizeAssistantResponse → sanitized=${result.was_modified}, warnings=${result.warnings.length}`+ logger.info(+ { component: 'Safety', sanitized: result.was_modified, warningCount: result.warnings.length },+ 'Assistant response sanitized')return result}@@ -254,12 +270,21 @@ export class BasicSafety implements Safety {*/wrapForLlm(toolName: string, content: string, wasSanitized: boolean): string {// Risk 4.10: Escape potential closing tags in content- const escaped = content.replace(/<\/tool_output>/gi, '</tool_output>')- const wrapped = `<tool_output name="${toolName}" sanitized="${wasSanitized}">\n${escaped}\n</tool_output>`- console.log(`[NewCore:Safety] wrapForLlm → tool=${toolName}, wrappedLength=${wrapped.length}`)+ const wrapped = wrapToolOutput(toolName, content, wasSanitized)+ logger.info(+ { component: 'Safety', toolName, wrappedLength: wrapped.length },+ 'Tool output wrapped'+ )return wrapped}+ previewOutputForLlm(toolName: string, content: string): string {+ const sanitized = this.sanitizeFreeformContent(content, {+ secretWarning: `Potential secret detected in ${toolName} output`,+ })+ return wrapToolOutput(toolName, sanitized.content, sanitized.was_modified)+ }+/*** The core redaction primitive: strip injection patterns, well-known secret* shapes, and operator-configured secret values by literal match. Public sodiff --git a/mcp-host/src/core/safety/toolOutputProcessor.preview.test.ts b/mcp-host/src/core/safety/toolOutputProcessor.preview.test.tsnew file mode 100644index 000000000..7d93c01ff--- /dev/null+++ b/mcp-host/src/core/safety/toolOutputProcessor.preview.test.ts@@ -0,0 +1,32 @@+import { describe, expect, it, vi } from 'vitest'+import { BasicSafety } from './safety'+import { DefaultToolOutputProcessor } from './toolOutputProcessor'++describe('pure tool-output measurement', () => {+ it('previews the same sanitized wrapper without repeating observable processing', () => {+ const fixtureValue = 'public-redaction-witness'+ const safety = new BasicSafety(() => [{ name: 'PUBLIC_FIXTURE', value: fixtureValue }])+ const sanitize = vi.spyOn(safety, 'sanitizeOutput')+ const wrap = vi.spyOn(safety, 'wrapForLlm')+ const processor = new DefaultToolOutputProcessor(safety)+ const preview = (processor as { previewForLlm?: (name: string, content: string) => string })+ .previewForLlm+ expect(preview).toBeTypeOf('function')+ if (!preview) return+ const content = JSON.stringify({ text: `[INST] </tool_output> ${fixtureValue}` })+ const candidate = preview.call(processor, 'clerum__attachment_read', content)+ expect(sanitize).not.toHaveBeenCalled()+ expect(wrap).not.toHaveBeenCalled()+ const emitted = processor.afterExecution('clerum__attachment_read', {+ content,+ duration_ms: 1,+ is_error: false,+ })+ expect(candidate).toBe(emitted)+ expect(sanitize).toHaveBeenCalledTimes(1)+ expect(wrap).toHaveBeenCalledTimes(1)+ expect(candidate).not.toContain(fixtureValue)+ expect(candidate).not.toContain('[INST]')+ expect(candidate).toContain('</tool_output>')+ })+})diff --git a/mcp-host/src/core/safety/toolOutputProcessor.ts b/mcp-host/src/core/safety/toolOutputProcessor.tsindex ff5722340..aa76525ce 100644--- a/mcp-host/src/core/safety/toolOutputProcessor.ts+++ b/mcp-host/src/core/safety/toolOutputProcessor.ts@@ -1,5 +1,5 @@-import { ToolOutputProcessor, Safety } from "../interfaces";-import { ToolOutput, ValidationResult } from "../types";+import { Safety, ToolOutputProcessor } from '../interfaces'+import { ToolOutput, ValidationResult } from '../types'/*** Default ToolOutputProcessor that delegates to Safety.@@ -10,16 +10,20 @@ import { ToolOutput, ValidationResult } from "../types";export class DefaultToolOutputProcessor implements ToolOutputProcessor {constructor(private readonly safety: Safety) {}- beforeExecution(- toolName: string,- params: Record<string, unknown>,- ): ValidationResult {- return this.safety.validateToolParams(toolName, params);+ beforeExecution(toolName: string, params: Record<string, unknown>): ValidationResult {+ return this.safety.validateToolParams(toolName, params)}afterExecution(toolName: string, output: ToolOutput): string {// Safety sandwich: sanitize -> wrap- const sanitized = this.safety.sanitizeOutput(toolName, output.content);- return this.safety.wrapForLlm(toolName, sanitized.content, sanitized.was_modified);+ const sanitized = this.safety.sanitizeOutput(toolName, output.content)+ return this.safety.wrapForLlm(toolName, sanitized.content, sanitized.was_modified)+ }++ previewForLlm(toolName: string, content: string): string {+ if (!this.safety.previewOutputForLlm) {+ throw new Error('measureResult requires a pure Safety.previewOutputForLlm implementation')+ }+ return this.safety.previewOutputForLlm(toolName, content)}}diff --git a/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.failure.test.ts b/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.failure.test.tsnew file mode 100644index 000000000..39935f496--- /dev/null+++ b/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.failure.test.ts@@ -0,0 +1,98 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'++const state = vi.hoisted(() => ({+ loadThrows: false,+ encodeThrows: false,+ encodeLengthThrows: false,+ metric: vi.fn(),+ warn: vi.fn(),+}))++vi.mock('../metrics', () => ({ tokenizerFallbackTotal: { inc: state.metric } }))+vi.mock('../../../logger', () => ({ logger: { warn: state.warn } }))++vi.mock('tiktoken', () => {+ const encoder = {+ encode: (text: string) => {+ if (state.encodeThrows) throw new Error('wasm crashed')+ // 1 token per UTF-16 unit: enough to tell the real path from the fallback.+ return new Uint32Array(text.length)+ },+ }+ return {+ get_encoding: () => {+ if (state.loadThrows) throw new Error('wasm unavailable')+ return encoder+ },+ encoding_for_model: () => encoder,+ }+})++vi.mock('../encodeLength', async importOriginal => {+ const actual = await importOriginal<typeof import('../encodeLength')>()+ return {+ ...actual,+ encodeLength: (...args: Parameters<typeof actual.encodeLength>) => {+ if (state.encodeLengthThrows) throw new Error('encode exploded')+ return actual.encodeLength(...args)+ },+ }+})++async function loadEstimate() {+ vi.resetModules()+ const { bpeTokenEstimate } = await import('../bpeEstimate')+ return bpeTokenEstimate+}++describe('bpeTokenEstimate — encoder failure fallback', () => {+ beforeEach(() => {+ state.loadThrows = false+ state.encodeThrows = false+ state.encodeLengthThrows = false+ state.metric.mockClear()+ state.warn.mockClear()+ })++ it('uses the real encoder while it works', async () => {+ const estimate = await loadEstimate()+ expect(estimate('abcd')).toBe(4)+ })++ it('returns the exact UTF-8 byte length when the encoder cannot load', async () => {+ state.loadThrows = true+ const estimate = await loadEstimate()+ const text = '日本語テキストです'+ expect(estimate(text)).toBe(Buffer.byteLength(text, 'utf8'))+ expect(estimate(text)).toBe(Buffer.byteLength(text, 'utf8'))+ expect(state.warn).toHaveBeenCalledTimes(1)+ expect(state.metric).toHaveBeenCalledWith({ provider: 'openai', reason: 'bpe_estimate_failed' })+ })++ it('returns the exact UTF-8 byte length when an encode fails', async () => {+ const estimate = await loadEstimate()+ state.encodeThrows = true+ const text = '😀🎉 emoji and more'+ expect(() => estimate(text)).not.toThrow()+ expect(estimate(text)).toBe(Buffer.byteLength(text, 'utf8'))+ state.encodeThrows = false+ expect(estimate('abcd')).toBe(4)+ })++ it('returns the exact UTF-8 byte length when the length helper throws and then recovers', async () => {+ const estimate = await loadEstimate()+ state.encodeLengthThrows = true+ const text = 'fallback please'+ expect(estimate(text)).toBe(Buffer.byteLength(text, 'utf8'))+ state.encodeLengthThrows = false+ expect(estimate('abcd')).toBe(4)+ })++ it('returns 0 for empty text on every path', async () => {+ const estimate = await loadEstimate()+ expect(estimate('')).toBe(0)+ state.loadThrows = true+ const failed = await loadEstimate()+ expect(failed('')).toBe(0)+ })+})diff --git a/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.test.ts b/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.test.tsnew file mode 100644index 000000000..2a2c44f89--- /dev/null+++ b/mcp-host/src/core/tokenizer/__tests__/bpeEstimate.test.ts@@ -0,0 +1,30 @@+import { describe, expect, it } from 'vitest'+import { bpeTokenEstimate } from '../bpeEstimate'++describe('bpeTokenEstimate', () => {+ it('counts dense, CJK and emoji text from the real encoder', () => {+ const cjk = '日本語テキストです'+ const dense = 'x'.repeat(4096)+ const emoji = '😀🎉'+ for (const text of [dense, cjk, emoji, cjk + emoji]) {+ const estimate = bpeTokenEstimate(text)+ // A real BPE count is positive and never exceeds the UTF-8 byte length,+ // which is the estimator's documented upper bound on the failure path.+ expect(estimate).toBeGreaterThan(0)+ expect(estimate).toBeLessThanOrEqual(Buffer.byteLength(text, 'utf8'))+ }+ })++ it('never splits a surrogate pair for a long emoji run', () => {+ const text = 'z'.repeat(1023) + '😀' + 'q'.repeat(2048)+ expect(bpeTokenEstimate(text)).toBeGreaterThan(0)+ })++ it('returns the exact UTF-8 byte length for a lone-surrogate input without throwing', () => {+ const lone = 'a\uD83Db'+ const bytes = Buffer.byteLength(lone, 'utf8')+ expect(() => bpeTokenEstimate(lone)).not.toThrow()+ expect(bpeTokenEstimate(lone)).toBeGreaterThan(0)+ expect(bpeTokenEstimate(lone)).toBeLessThanOrEqual(bytes)+ })+})diff --git a/mcp-host/src/core/tokenizer/__tests__/encodeLength.test.ts b/mcp-host/src/core/tokenizer/__tests__/encodeLength.test.tsnew file mode 100644index 000000000..6f7bf6d74--- /dev/null+++ b/mcp-host/src/core/tokenizer/__tests__/encodeLength.test.ts@@ -0,0 +1,143 @@+import { describe, expect, it, vi } from 'vitest'+import { type Tiktoken, get_encoding } from 'tiktoken'+import { ENCODE_SLICE_UTF16_UNITS, encodeLength } from '../encodeLength'++const encoder = get_encoding('cl100k_base')++describe('encodeLength', () => {+ it('treats special-token syntax as ordinary text instead of throwing', () => {+ const text = 'before <|endoftext|> after'+ // Raw tiktoken rejects this shape; the helper must count it.+ expect(() => encoder.encode(text)).toThrow(/special token/)+ expect(encodeLength(encoder, text)).toBe(encoder.encode(text, [], []).length)+ })++ it('returns 0 for the empty string', () => {+ expect(encodeLength(encoder, '')).toBe(0)+ })++ it('matches whole-string encoding for long ASCII runs', () => {+ const text = 'a'.repeat(10_000)+ expect(encodeLength(encoder, text)).toBe(encoder.encode(text, [], []).length)+ })++ it('keeps the token count at a slice boundary', () => {+ const boundary = ENCODE_SLICE_UTF16_UNITS+ for (const text of [+ 'x'.repeat(boundary),+ 'x'.repeat(boundary + 1),+ 'x'.repeat(boundary - 1),+ 'y'.repeat(boundary * 3 + 7),+ ]) {+ expect(encodeLength(encoder, text)).toBe(encoder.encode(text, [], []).length)+ }+ })++ it('never splits a surrogate pair at the slice boundary', () => {+ // Position the emoji so its high surrogate is the last unit of slice 1.+ const text = 'z'.repeat(ENCODE_SLICE_UTF16_UNITS - 1) + '😀' + 'q'.repeat(64)+ const sliced = encodeLength(encoder, text)+ expect(sliced).toBe(encoder.encode(text, [], []).length)+ // An unpaired surrogate would be replaced with U+FFFD and change the count.+ expect(sliced).toBeGreaterThan(0)+ expect(sliced).not.toBe(encodeLength(encoder, text.replace('😀', '\uFFFD')))+ })++ it('never under-counts a separator-free mix of CJK, emoji and dense runs', () => {+ const dense = 'x'.repeat(ENCODE_SLICE_UTF16_UNITS * 2)+ const cjk = '日本語テキストです'.repeat(40)+ const emoji = '😀🎉🚀'.repeat(37)+ for (const text of [dense, cjk, emoji, cjk + emoji + dense]) {+ const whole = encoder.encode(text, [], []).length+ const sliced = encodeLength(encoder, text)+ // A hard cut inside a dense run may over-count by a token or two; it must+ // never lose tokens, because an under-count is what overflows a budget.+ expect(sliced).toBeGreaterThanOrEqual(whole)+ expect(sliced - whole).toBeLessThanOrEqual(2)+ }+ })++ it('matches whole-string encoding exactly when every chunk ends at a clean line break', () => {+ const lines: string[] = []+ for (let i = 0; i < 80; i++) lines.push(`row-${i.toString().padStart(4, '0')}=ok`)+ const text = `${lines.join('\n')}\n`+ expect(text.length).toBeLessThan(ENCODE_SLICE_UTF16_UNITS)+ expect(encodeLength(encoder, text)).toBe(encoder.encode(text, [], []).length)+ const repeated = text.repeat(3)+ expect(repeated.length).toBeGreaterThan(ENCODE_SLICE_UTF16_UNITS)+ expect(encodeLength(encoder, repeated)).toBe(encoder.encode(repeated, [], []).length)+ })++ it('stays within two tokens of whole-string encoding on separated real text', () => {+ const lines: string[] = []+ for (let i = 0; i < 120; i++) {+ lines.push(`line ${i}: 日本語テキスト 😀🎉 ${'q'.repeat(24)}`)+ }+ const text = `${lines.join('\n')}\n`+ expect(text.length).toBeGreaterThan(ENCODE_SLICE_UTF16_UNITS * 2)+ const whole = encoder.encode(text, [], []).length+ const sliced = encodeLength(encoder, text)+ // Greedy pre-tokenization at a cut can over-count by one token. The+ // bounded guarantee for real text is: never lose tokens, drift <= 2.+ expect(sliced).toBeGreaterThanOrEqual(whole)+ expect(sliced - whole).toBeLessThanOrEqual(2)+ })++ it('keeps every slice at or below the UTF-16 bound', () => {+ const dense = 'x'.repeat(ENCODE_SLICE_UTF16_UNITS * 5 + 17)+ const text = `head\n${dense}\ntail`+ const encode = vi.fn((slice: string) => new Uint32Array(slice.length))+ expect(encodeLength({ encode } as unknown as Tiktoken, text)).toBe(text.length)+ expect(encode.mock.calls.length).toBeGreaterThan(1)+ for (const [slice] of encode.mock.calls) {+ expect(slice.length).toBeLessThanOrEqual(ENCODE_SLICE_UTF16_UNITS)+ }+ })++ it('passes complete surrogate pairs and explicit special-token options to each real call', () => {+ const text = 'z'.repeat(1023) + '😀' + 'x'.repeat(4096) + '<|endoftext|>'+ const encode = vi.fn((slice: string, allowed: string[], disallowed: string[]) => {+ expect(new TextDecoder().decode(new TextEncoder().encode(slice))).toBe(slice)+ expect(allowed).toEqual([])+ expect(disallowed).toEqual([])+ return new Uint32Array(1)+ })+ expect(encodeLength({ encode } as unknown as Tiktoken, text)).toBeGreaterThan(1)+ expect(encode.mock.calls.map(([slice]) => slice).join('')).toBe(text)+ })++ it('matches whole-string encoding exactly for line-oriented content', () => {+ const logs =+ '2026-10-02T19:00:00Z INFO host=control-api msg="request completed" status=200 ms=42 path=/v1/runtime\n'.repeat(+ 500+ )+ const fences = '```ts\nconst x: number = 1\nexport { x }\n```\n'.repeat(300)+ const jsonl = Array.from(+ { length: 500 },+ (_, i) => `{"i":${i},"kind":"page","bytes":65536,"ok":true}`+ )+ .join('\n')+ .concat('\n')+ for (const text of [logs, fences, jsonl]) {+ expect(text.length).toBeGreaterThan(ENCODE_SLICE_UTF16_UNITS * 2)+ expect(encodeLength(encoder, text)).toBe(encoder.encode(text, [], []).length)+ }+ })++ it('stays under 1% over-count on prose with multibyte characters', () => {+ const proseEs =+ 'La plataforma procesa documentos adjuntos y los pagina por rango de bytes sin romper caracteres multibyte. '.repeat(+ 90+ )+ const proseEn =+ 'The service streams attachment pages in bounded windows and never splits a multi byte character or a surrogate pair. '.repeat(+ 70+ )+ for (const text of [proseEs, proseEn]) {+ const whole = encoder.encode(text, [], []).length+ const sliced = encodeLength(encoder, text)+ expect(sliced).toBeGreaterThanOrEqual(whole)+ expect((sliced - whole) / whole).toBeLessThan(0.01)+ }+ })+})diff --git a/mcp-host/src/core/tokenizer/__tests__/openaiTokenCounter.failure.test.ts b/mcp-host/src/core/tokenizer/__tests__/openaiTokenCounter.failure.test.tsnew file mode 100644index 000000000..537c0738e--- /dev/null+++ b/mcp-host/src/core/tokenizer/__tests__/openaiTokenCounter.failure.test.ts@@ -0,0 +1,121 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import type { ChatMessage, ToolDefinition } from '../../types'++const state = vi.hoisted(() => ({+ encodeThrows: false,+ loadThrows: false,+}))++vi.mock('tiktoken', () => {+ const encoder = {+ encode: () => {+ if (state.encodeThrows) throw new Error('wasm crashed')+ return new Uint32Array(3)+ },+ }+ return {+ encoding_for_model: () => {+ if (state.loadThrows) throw new Error('unknown model')+ return encoder+ },+ get_encoding: () => {+ if (state.loadThrows) throw new Error('wasm unavailable')+ return encoder+ },+ }+})++const MESSAGE: ChatMessage = { role: 'user', content: '日本語テキストです' }++function exactUtf8Bytes(text: string): number {+ return Buffer.byteLength(text, 'utf8')+}++/** The framing the healthy path always adds: 3 per message + 3 trailing prime. */+function framing(messageCount: number): number {+ return messageCount * 3 + 3+}++async function createCounter(model: string) {+ // Dynamic import AFTER `vi.mock` installs the controllable encoder.+ const { OpenAITokenCounter } = await import('../openaiTokenCounter')+ return new OpenAITokenCounter(model)+}++describe('OpenAITokenCounter — encoder failure fallback', () => {+ beforeEach(() => {+ state.encodeThrows = false+ state.loadThrows = false+ })++ it('counts normally while the mocked encoder works', async () => {+ const counter = await createCounter('gpt-4o-mini')+ await counter.warmup()+ // 3 (message overhead) + 3 (encoded content) + 3 (trailing prime).+ expect(counter.countSync([MESSAGE])).toBe(9)+ await expect(counter.count([MESSAGE])).resolves.toBe(9)+ })++ it('returns the exact UTF-8 byte length instead of throwing when encode fails', async () => {+ const counter = await createCounter('gpt-4o-mini')+ await counter.warmup()+ state.encodeThrows = true+ expect(() => counter.countSync([MESSAGE])).not.toThrow()+ expect(counter.countSync([MESSAGE])).toBe(exactUtf8Bytes(MESSAGE.content) + framing(1))+ await expect(counter.count([MESSAGE])).resolves.toBe(+ exactUtf8Bytes(MESSAGE.content) + framing(1)+ )+ })++ it('keeps the message framing on the fallback path, including empty content', async () => {+ const counter = await createCounter('gpt-4o-mini')+ await counter.warmup()+ state.encodeThrows = true+ const empty: ChatMessage = { role: 'user', content: '' }+ // The framing is real request weight; a zero here would undercount every+ // budget check that runs while the encoder is unavailable.+ expect(counter.countSync([empty])).toBe(framing(1))+ expect(counter.countSync([empty, empty])).toBe(framing(2))+ expect(counter.countSync([empty])).toBeGreaterThan(0)+ })++ it('counts tool schemas in the byte-length fallback', async () => {+ const counter = await createCounter('gpt-4o-mini')+ await counter.warmup()+ state.encodeThrows = true+ const tools: ToolDefinition[] = [+ { name: 'tool_one', description: 'first tool', parameters: { type: 'object' } },+ ]+ const expected =+ exactUtf8Bytes(MESSAGE.content) ++ framing(1) ++ exactUtf8Bytes(tools[0].name) ++ exactUtf8Bytes(tools[0].description ?? '') ++ exactUtf8Bytes(JSON.stringify(tools[0].parameters))+ expect(counter.countSync([MESSAGE], tools)).toBe(expected)+ })++ it('resolves warmup without throwing when no encoder can be loaded', async () => {+ state.loadThrows = true+ const counter = await createCounter('gpt-4o-mini')+ await expect(counter.warmup()).resolves.toBeUndefined()+ // The counter stays usable after the failed load, not stuck or rejected.+ expect(counter.countSync([MESSAGE])).toBe(exactUtf8Bytes(MESSAGE.content) + framing(1))+ })++ it('cannot treat an unserializable tool schema as free while the encoder is unavailable', async () => {+ const counter = await createCounter('gpt-4o-mini')+ await counter.warmup()+ state.encodeThrows = true+ const parameters: Record<string, unknown> = {}+ parameters.circular = parameters+ const valid: ToolDefinition = {+ name: 'bounded_tool',+ description: '',+ parameters: { type: 'object' },+ }+ expect(counter.countSync([MESSAGE], [valid])).toBeGreaterThan(framing(1))+ const invalid = { ...valid, parameters }+ expect(counter.countSync([MESSAGE], [invalid])).toBe(Number.MAX_SAFE_INTEGER)+ })+})diff --git a/mcp-host/src/core/tokenizer/bpeEstimate.ts b/mcp-host/src/core/tokenizer/bpeEstimate.tsnew file mode 100644index 000000000..111b717bc--- /dev/null+++ b/mcp-host/src/core/tokenizer/bpeEstimate.ts@@ -0,0 +1,66 @@+/**+ * BPE token estimate for the OpenAI-family tokenizer.+ *+ * Healthy estimates use bounded cl100k_base encoding. Encoder failures use+ * the full UTF-8 byte length, including zero for empty text, without dividing+ * by a guessed bytes-per-token ratio.+ *+ * The cl100k_base encoder is loaded lazily on first use, from the same+ * `get_encoding` helper `OpenAITokenCounter` already imports, and reused for+ * the life of the process. A load or encode failure never throws and never+ * aborts startup: the exact byte length is returned, one bounded metric+ * counter and one structured warning record the reason. A failed initial load+ * keeps using the byte bound for this process rather than repeatedly loading+ * an unavailable encoder.+ */+import { type Tiktoken, get_encoding } from 'tiktoken'+import { logger } from '../../logger'+import { encodeLength } from './encodeLength'+import { tokenizerFallbackTotal } from './metrics'++let encoder: Tiktoken | null = null+let encoderLoadFailed = false+let warned = false++/** Fixed reason values: never an exception name and never text content. */+export type BpeFallbackReason = 'encoder_load_failed' | 'encode_failed'++function warnOnce(reason: BpeFallbackReason): void {+ if (warned) return+ warned = true+ logger.warn(+ { component: 'BpeEstimate', reason },+ 'BPE token estimate unavailable; using UTF-8 byte length'+ )+}++function getEncoder(): Tiktoken | null {+ if (encoder) return encoder+ if (encoderLoadFailed) return null+ try {+ encoder = get_encoding('cl100k_base')+ return encoder+ } catch {+ encoderLoadFailed = true+ tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'bpe_estimate_failed' })+ warnOnce('encoder_load_failed')+ return null+ }+}++/**+ * Token estimate for `text` under cl100k_base, with the exact UTF-8 byte+ * length as the documented conservative fallback.+ */+export function bpeTokenEstimate(text: string): number {+ if (text.length === 0) return 0+ const encoderOrNull = getEncoder()+ if (!encoderOrNull) return Buffer.byteLength(text, 'utf8')+ try {+ return encodeLength(encoderOrNull, text)+ } catch {+ tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'bpe_estimate_failed' })+ warnOnce('encode_failed')+ return Buffer.byteLength(text, 'utf8')+ }+}diff --git a/mcp-host/src/core/tokenizer/encodeLength.ts b/mcp-host/src/core/tokenizer/encodeLength.tsnew file mode 100644index 000000000..f94fe59e8--- /dev/null+++ b/mcp-host/src/core/tokenizer/encodeLength.ts@@ -0,0 +1,100 @@+/**+ * UTF-16-safe encode-length helper shared by the tokenizer implementations.+ *+ * Two constraints shape this helper:+ *+ * 1. `encode()` with the default arguments REFUSES the special-token syntax+ * (`<|endoftext|>`, `<|fim_prefix|>`, …) and throws+ * `The text contains a special token that is not allowed`. Tool output is+ * untrusted, so a page that happens to carry that literal would otherwise+ * turn a token count into a task failure. Passing `[]` for+ * `allowedSpecial`/`disallowedSpecial` lets tiktoken encode the syntax as+ * ordinary text instead of rejecting the request.+ *+ * 2. A single call over a whole attachment page feeds one unbounded input into+ * the WASM encoder. Slicing at 1024 UTF-16 units bounds the per-call input;+ * throughput is comparable to one whole-string call, so the slicing is+ * about the bound, not about speed.+ *+ * BPE counts can differ across slice boundaries. Aligning a cut with a nearby+ * line break or space reduces that difference for prose and line-based data.+ * The lookback is bounded; the caller still measures the whole emitted message+ * with its applicable counters rather than assuming a byte-to-token ratio.+ *+ * A slice boundary must never split a surrogate pair: `str.slice` on a lone+ * high surrogate produces an unpaired code unit, which the encoder replaces+ * with U+FFFD and the count drifts. The adjuster below moves the boundary back+ * one unit when the cut would land between a high and a low surrogate.+ */+import type { Tiktoken } from 'tiktoken'++/** UTF-16 code units per encode call. See the file header for the rationale. */+export const ENCODE_SLICE_UTF16_UNITS = 1024++/**+ * How far back the aligner may walk to find a separator. Bounded so a page of+ * dense characters still encodes in O(n) slices instead of degenerating into a+ * single full-page call.+ */+export const ENCODE_BOUNDARY_LOOKBACK_UTF16_UNITS = 256++function isHighSurrogate(code: number): boolean {+ return code >= 0xd800 && code <= 0xdbff+}++function isLowSurrogate(code: number): boolean {+ return code >= 0xdc00 && code <= 0xdfff+}++function isLineBreak(code: number): boolean {+ return code === 0x0a || code === 0x0d+}++function isInlineSeparator(code: number): boolean {+ return code === 0x20 || code === 0x09+}++/**+ * Position <= `nominalEnd` where the next slice can start cleanly, or+ * `nominalEnd` itself when the bounded window holds no separator. Line breaks+ * win over spaces to keep complete lines together when possible; within one+ * class the nearest candidate wins.+ */+function alignSliceEnd(text: string, start: number, nominalEnd: number): number {+ const floor = Math.max(start + 1, nominalEnd - ENCODE_BOUNDARY_LOOKBACK_UTF16_UNITS)+ let inlineCandidate = nominalEnd+ for (let end = nominalEnd; end > floor; end--) {+ const code = text.charCodeAt(end - 1)+ if (isLineBreak(code)) return end+ if (inlineCandidate === nominalEnd && isInlineSeparator(code)) inlineCandidate = end+ }+ return inlineCandidate+}++/**+ * Number of tokens `encoder` assigns to `text`.+ *+ * Encodes in bounded UTF-16 slices, never splitting a surrogate pair, and+ * always with special-token syntax treated as ordinary text. Throws only what+ * the encoder itself throws; callers that must not fail own the fallback.+ */+export function encodeLength(encoder: Tiktoken, text: string): number {+ if (text.length === 0) return 0+ let total = 0+ let start = 0+ while (start < text.length) {+ const nominalEnd = Math.min(start + ENCODE_SLICE_UTF16_UNITS, text.length)+ let end = nominalEnd+ if (nominalEnd < text.length) end = alignSliceEnd(text, start, nominalEnd)+ if (+ end < text.length &&+ isHighSurrogate(text.charCodeAt(end - 1)) &&+ isLowSurrogate(text.charCodeAt(end))+ ) {+ end -= 1+ }+ total += encoder.encode(text.slice(start, end), [], []).length+ start = end+ }+ return total+}diff --git a/mcp-host/src/core/tokenizer/metrics.ts b/mcp-host/src/core/tokenizer/metrics.tsindex 41c607a9b..0837716f3 100644--- a/mcp-host/src/core/tokenizer/metrics.ts+++ b/mcp-host/src/core/tokenizer/metrics.ts@@ -6,7 +6,8 @@* Labels:* - provider: any registered LlmProvider id (see llm/registryCore.ts)* - reason: 'no_native_api' | 'rate_limit' | 'count_call_failed' |- * 'unknown_model' | 'offline' | 'no_warmup'+ * 'unknown_model' | 'offline' | 'no_warmup' |+ * 'bpe_estimate_failed'* - tier_chosen / from / to: 'passthrough' | 'workspace' | 'summarize' | 'truncate'*/import { Counter, Histogram } from 'prom-client'@@ -44,3 +45,4 @@ export type TokenizerFallbackReason =| 'unknown_model'| 'offline'| 'no_warmup'+ | 'bpe_estimate_failed'diff --git a/mcp-host/src/core/tokenizer/openaiTokenCounter.ts b/mcp-host/src/core/tokenizer/openaiTokenCounter.tsindex 11f9a78cd..0052cc5d1 100644--- a/mcp-host/src/core/tokenizer/openaiTokenCounter.ts+++ b/mcp-host/src/core/tokenizer/openaiTokenCounter.ts@@ -10,13 +10,80 @@* gpt-4o families) with a warning. Images are NOT counted yet — the helper* `accountForImages` is reserved for a follow-up PR once Desktop multi-image* traffic is meaningful.+ *+ * Every encode goes through `encodeLength` (bounded slices, surrogate-safe,+ * special-token syntax treated as ordinary text) so untrusted tool output can+ * never make a count throw on `<|endoftext|>`. When an encode still fails, the+ * count falls back to the exact UTF-8 byte length of the same fields — never+ * `bytes / 4`, never `0` — and records one bounded metric plus one structured+ * warning.*/import { type Tiktoken, encoding_for_model, get_encoding } from 'tiktoken'+import { logger } from '../../logger'import type { ChatMessage, ToolDefinition } from '../types'-import { heuristicCount } from './heuristic'+import { encodeLength } from './encodeLength'import { tokenizerFallbackTotal } from './metrics'import type { TokenCounter } from './tokenCounter'+/**+ * One-shot guard for the encode-failure warning. `countSync` runs on every+ * pressure decision, so an encoder that stays broken must not log per call.+ */+let encodeFailureWarned = false++/** Fixed reason values: never an exception name, never model or text content. */+export type OpenAiCountFallbackReason = 'no_warmup' | 'encoder_load_failed' | 'encode_failed'++function warnEncodeFailureOnce(reason: OpenAiCountFallbackReason): void {+ if (encodeFailureWarned) return+ encodeFailureWarned = true+ logger.warn(+ { component: 'OpenAITokenCounter', reason },+ 'tiktoken encode failed; using the exact UTF-8 byte length as the conservative count'+ )+}++/** An unserializable payload cannot be priced or emitted; saturate the bound. */+function safeJsonBytes(value: unknown): number {+ try {+ const serialized = JSON.stringify(value)+ return serialized === undefined+ ? Number.MAX_SAFE_INTEGER+ : Buffer.byteLength(serialized, 'utf8')+ } catch {+ return Number.MAX_SAFE_INTEGER+ }+}++/**+ * Conservative fallback for a failed encode: the message framing the healthy+ * path always adds (3 per message + 3 trailing prime) plus the exact UTF-8 byte+ * length of every field the healthy path counts. Never `bytes / 4`, and never+ * zero for a message list, because the framing is real request weight.+ */+function byteLengthFallback(messages: ChatMessage[], tools: ToolDefinition[]): number {+ let total = 3 // trailing prime, same as the healthy path+ for (const msg of messages) {+ total += 3 // per-message overhead (role + separators)+ total += Buffer.byteLength(msg.content ?? '', 'utf8')+ if (msg.name) total += Buffer.byteLength(msg.name, 'utf8')+ if (msg.tool_call_id) total += Buffer.byteLength(msg.tool_call_id, 'utf8')+ for (const tc of msg.tool_calls ?? []) {+ total += Buffer.byteLength(tc.id, 'utf8')+ total += Buffer.byteLength(tc.name, 'utf8')+ total += safeJsonBytes(tc.arguments ?? {})+ }+ }+ for (const t of tools) {+ total += Buffer.byteLength(t.name, 'utf8')+ total += Buffer.byteLength(t.description ?? '', 'utf8')+ total += safeJsonBytes(t.parameters ?? {})+ }+ return Math.min(total, Number.MAX_SAFE_INTEGER)+}++export { encodeLength } from './encodeLength'+export class OpenAITokenCounter implements TokenCounter {readonly providerName = 'openai' as constprivate encoder: Tiktoken | null = null@@ -32,10 +99,21 @@ export class OpenAITokenCounter implements TokenCounter {this.encoder = encoding_for_model(this.modelName as Parameters<typeof encoding_for_model>[0])} catch {tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'unknown_model' })- console.warn(- `[TokenCounter:openai] model "${this.modelName}" unknown to tiktoken; using cl100k_base`+ // The model string is request-controlled, so the record names only the+ // fixed reason; the metric carries the bounded provider label.+ logger.warn(+ { component: 'OpenAITokenCounter', reason: 'unknown_model' },+ 'Model unknown to tiktoken; using cl100k_base')- this.encoder = get_encoding('cl100k_base')+ try {+ this.encoder = get_encoding('cl100k_base')+ } catch {+ // Letting this escape would reject `warmup()` and make the counter+ // unusable on a host whose WASM encoding is unavailable. `countSync`+ // below answers with the conservative byte-length bound instead.+ tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'count_call_failed' })+ warnEncodeFailureOnce('encoder_load_failed')+ }}}@@ -47,29 +125,19 @@ export class OpenAITokenCounter implements TokenCounter {countSync(messages: ChatMessage[], tools: ToolDefinition[] = []): number {if (!this.encoder) {tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'no_warmup' })- return Math.ceil(heuristicCount(messages) * 1.5)+ warnEncodeFailureOnce('no_warmup')+ return byteLengthFallback(messages, tools)}- let total = 0- for (const msg of messages) {- total += 3 // per-message overhead (role + separators)- total += this.encoder.encode(msg.content ?? '').length- if (msg.name) total += this.encoder.encode(msg.name).length- if (msg.tool_call_id) total += this.encoder.encode(msg.tool_call_id).length- if (msg.tool_calls) {- for (const tc of msg.tool_calls) {- total += this.encoder.encode(tc.id).length- total += this.encoder.encode(tc.name).length- total += this.encoder.encode(JSON.stringify(tc.arguments ?? {})).length- }- }- }- for (const t of tools) {- total += this.encoder.encode(t.name).length- total += this.encoder.encode(t.description ?? '').length- total += this.encoder.encode(JSON.stringify(t.parameters ?? {})).length+ const encoder = this.encoder+ try {+ return countWithEncoder(encoder, messages, tools)+ } catch {+ // A broken or partially-loaded encoder must not throw from a count: the+ // conservative byte-length bound keeps every budget check usable.+ tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'count_call_failed' })+ warnEncodeFailureOnce('encode_failed')+ return byteLengthFallback(messages, tools)}- total += 3 // trailing prime- return total}recordObservedUsage(usage: { input_tokens: number; output_tokens: number }): void {@@ -80,3 +148,31 @@ export class OpenAITokenCounter implements TokenCounter {return this.observed}}++function countWithEncoder(+ encoder: Tiktoken,+ messages: ChatMessage[],+ tools: ToolDefinition[]+): number {+ let total = 0+ for (const msg of messages) {+ total += 3 // per-message overhead (role + separators)+ total += encodeLength(encoder, msg.content ?? '')+ if (msg.name) total += encodeLength(encoder, msg.name)+ if (msg.tool_call_id) total += encodeLength(encoder, msg.tool_call_id)+ if (msg.tool_calls) {+ for (const tc of msg.tool_calls) {+ total += encodeLength(encoder, tc.id)+ total += encodeLength(encoder, tc.name)+ total += encodeLength(encoder, JSON.stringify(tc.arguments ?? {}))+ }+ }+ }+ for (const t of tools) {+ total += encodeLength(encoder, t.name)+ total += encodeLength(encoder, t.description ?? '')+ total += encodeLength(encoder, JSON.stringify(t.parameters ?? {}))+ }+ total += 3 // trailing prime+ return total+}diff --git a/mcp-host/src/core/tools/__tests__/attachmentRead.budget.test.ts b/mcp-host/src/core/tools/__tests__/attachmentRead.budget.test.tsnew file mode 100644index 000000000..79e37485d--- /dev/null+++ b/mcp-host/src/core/tools/__tests__/attachmentRead.budget.test.ts@@ -0,0 +1,350 @@+/**+ * C16 — current-turn budget fencing for `clerum__attachment_read`.+ *+ * Measurement mirrors production: BasicSafety's pure sanitize+wrap preview,+ * then a BPE-style bytes/4 estimate, so JSON quoting growth is visible to the+ * fitter. Every test drives the real execute -> finalize contract once.+ */+import { describe, expect, it } from 'vitest'+import { createHash } from 'node:crypto'+import { validateIncomingAttachments } from '../../../agent/incomingAttachments'+import type { IncomingMessage } from '../../../server'+import { AttachmentReadLedger } from '../../attachments/attachmentReadBudget'+import { BasicSafety } from '../../safety/safety'+import type { Attachment, ToolResult } from '../../types'+import { AttachmentReadTool } from '../attachmentRead'++const TOOL_NAME = 'clerum__attachment_read'+const READ_LIMIT = 65_536+const WINDOW_TOKENS = 20_000+const safety = new BasicSafety()++const bpe = (content: string): number => Math.ceil(Buffer.byteLength(content, 'utf8') / 4) + 4+const renderContent = (raw: string): string => safety.previewOutputForLlm(TOOL_NAME, raw)+const measureContent = (finalContent: string): number => bpe(finalContent)+const measureResult = (raw: string): number => measureContent(renderContent(raw))++function rawFile(id: string, filename: string, mimeType: string, bytes: Buffer) {+ return {+ id,+ kind: 'file',+ mimeType,+ detectedMediaType: mimeType,+ encoding: 'base64',+ dataBase64: bytes.toString('base64'),+ filename,+ sizeBytes: bytes.length,+ digest: { algorithm: 'sha256', hex: createHash('sha256').update(bytes).digest('hex') },+ }+}++function admitted(raw: unknown[]): Attachment[] {+ const result = validateIncomingAttachments(raw, {+ maxCount: 20,+ maxBytes: 4_000_000,+ maxFileBytes: 4_000_000,+ messageId: 'message-budget',+ })+ if (!result.ok) throw new Error(`fixture rejected: ${result.error.code}`)+ return result.attachments!+}++function messageFor(attachments: Attachment[]): IncomingMessage {+ return {+ content: 'Read the attached file',+ channelType: 'rpc',+ channelId: 'agent-budget',+ sender: 'user-budget',+ timestamp: '2026-10-02T10:00:00Z',+ messageId: 'message-budget',+ hostRef: 'host-budget',+ attachments,+ }+}++function toolFor(+ attachments: Attachment[],+ options: { ledger?: AttachmentReadLedger; windowTokens?: number } = {}+) {+ const ledger = options.ledger ?? new AttachmentReadLedger()+ return {+ tool: new AttachmentReadTool(messageFor(attachments), READ_LIMIT, {+ contextWindowTokens: options.windowTokens ?? WINDOW_TOKENS,+ ledger,+ }),+ ledger,+ }+}++interface RunResult {+ outputBody: Record<string, unknown>+ finalized: ToolResult+ rawOutput: string+}++async function runRead(+ tool: AttachmentReadTool,+ params: Record<string, unknown>,+ options: { transform?: (content: string) => string } = {}+): Promise<RunResult> {+ const output = await tool.execute(params, { onOutput: () => {}, measureResult })+ let content = renderContent(output.content)+ if (options.transform) content = options.transform(content)+ const result: ToolResult = {+ tool_call_id: 'tc_budget',+ name: TOOL_NAME,+ content,+ is_error: output.is_error,+ metadata: output.metadata,+ rawContent: output.content,+ emittedMessageCost: measureContent(content),+ }+ const finalized = tool.finalizeResult!(result, { measureContent, renderContent })+ return {+ outputBody: JSON.parse(output.content) as Record<string, unknown>,+ finalized,+ rawOutput: output.content,+ }+}++describe('C16 constructor contract', () => {+ it('requires the budget options as a mandatory third constructor argument', () => {+ expect(() => new AttachmentReadTool(messageFor([]), READ_LIMIT, undefined as never)).toThrow(+ /attachment read options/i+ )+ })+})++describe('C16 page fitting', () => {+ it('walks CJK and emoji pages through nextOffset without splitting code points', async () => {+ const original = `漢字📚${'😀'.repeat(20_000)}fin`+ const bytes = Buffer.from(original, 'utf8')+ const [file] = admitted([rawFile('file-cjk', 'walk.txt', 'text/plain', bytes)])+ const { tool } = toolFor([file!], { windowTokens: 100_000 })++ const parts: string[] = []+ const origins: string[] = []+ let offset = 0+ let guard = 0+ while (guard++ < 100) {+ const { outputBody, finalized } = await runRead(tool, { attachmentId: 'file-cjk', offset })+ const range = outputBody.byteRange as { offset: number; length: number }+ expect(range.offset).toBe(offset)+ expect(range.length).toBeGreaterThan(0)+ parts.push(outputBody.text as string)+ origins.push(+ ((finalized.metadata as Record<string, unknown>).attachmentRead as Record<string, unknown>)+ .origin as string+ )+ offset += range.length+ if (outputBody.truncated === true) {+ expect(outputBody.nextOffset).toBe(offset)+ expect(typeof outputBody.limit).toBe('string')+ } else {+ expect(outputBody.nextOffset).toBeUndefined()+ expect(outputBody.limit).toBeUndefined()+ break+ }+ }+ expect(parts.join('')).toBe(original)+ expect(parts.join('')).not.toContain('\uFFFD')+ expect(origins[0]).toBe('current')+ expect(origins[1]).toBe('current_resumed')+ })++ it('shrinks quote-dense JSON more than plain ASCII under the real wrapper measurement', async () => {+ const dense = '"'.repeat(120_000)+ const plain = 'a'.repeat(120_000)+ const [denseFile, plainFile] = admitted([+ rawFile('file-dense', 'dense.json', 'application/json', Buffer.from(dense)),+ rawFile('file-plain', 'plain.txt', 'text/plain', Buffer.from(plain)),+ ])+ const denseTool = toolFor([denseFile!]).tool+ const plainTool = toolFor([plainFile!]).tool++ const densePage = await runRead(denseTool, { attachmentId: 'file-dense' })+ const plainPage = await runRead(plainTool, { attachmentId: 'file-plain' })+ const denseRange = densePage.outputBody.byteRange as { length: number }+ const plainRange = plainPage.outputBody.byteRange as { length: number }+ // Quotes double in JSON (" -> \"); the fitter must see that growth.+ expect(denseRange.length).toBeLessThan(plainRange.length)+ expect(densePage.outputBody.limit).toBe('page_budget')+ expect(plainPage.outputBody.limit).toBe('page_budget')+ })++ it('caps a page by the 10% page budget', async () => {+ const [file] = admitted([+ rawFile('file-page', 'page.txt', 'text/plain', Buffer.alloc(120_000, 'x')),+ ])+ const { tool, ledger } = toolFor([file!])+ const before = ledger.snapshot()+ const { outputBody, finalized } = await runRead(tool, { attachmentId: 'file-page' })+ const range = outputBody.byteRange as { length: number }++ expect(outputBody.limit).toBe('page_budget')+ expect(range.length).toBeLessThan(READ_LIMIT)+ expect(finalized.emittedMessageCost).toBeLessThanOrEqual(Math.floor(WINDOW_TOKENS * 0.1))+ expect(ledger.snapshot().spentTokens).toBeGreaterThan(before.spentTokens)+ expect(ledger.snapshot().bytesRead).toBe(range.length)+ })++ it('caps later pages by the remaining 30% turn budget', async () => {+ const [file] = admitted([+ rawFile('file-turn', 'turn.txt', 'text/plain', Buffer.alloc(400_000, 'y')),+ ])+ const { tool, ledger } = toolFor([file!])++ let offset = 0+ const limits: string[] = []+ for (let page = 0; page < 3; page++) {+ const { outputBody } = await runRead(tool, { attachmentId: 'file-turn', offset })+ limits.push(outputBody.limit as string)+ offset += (outputBody.byteRange as { length: number }).length+ }+ expect(limits[0]).toBe('page_budget')+ expect(limits[limits.length - 1]).toBe('turn_budget')+ expect(ledger.snapshot().spentTokens).toBeLessThanOrEqual(Math.floor(WINDOW_TOKENS * 0.3))+ })+})++describe('C16 read count and exhaustion', () => {+ it('allows the 32nd read and answers the 33rd with a success notice', async () => {+ const [file] = admitted([+ rawFile('file-32', 'count.txt', 'text/plain', Buffer.from('countable page body')),+ ])+ const { tool, ledger } = toolFor([file!])++ for (let call = 1; call <= 32; call++) {+ const { outputBody } = await runRead(tool, { attachmentId: 'file-32', maxBytes: 4 })+ expect(outputBody.kind).toBe('text')+ }+ expect(ledger.snapshot().reads).toBe(32)++ const thirtyThird = await runRead(tool, { attachmentId: 'file-32', offset: 4 })+ expect(thirtyThird.finalized.is_error).toBe(false)+ expect(thirtyThird.outputBody.kind).toBe('read_budget_exhausted')+ expect(thirtyThird.outputBody.resumeOffset).toBe(4)+ const message = thirtyThird.outputBody.message as string+ expect(message).toMatch(/new message/i)+ expect(message).toMatch(/current-turn|current turn/i)+ expect(message).not.toMatch(/earlier (pages )?(are |is )?stored/i)+ expect(thirtyThird.rawOutput).not.toContain('countable')+ expect(ledger.snapshot().reads).toBe(32)+ })++ it('charges native errors in finalize', async () => {+ const [file] = admitted([rawFile('file-err', 'errors.txt', 'text/plain', Buffer.from('aé!'))])+ const { tool, ledger } = toolFor([file!])+ const before = ledger.snapshot()++ const bad = await runRead(tool, { attachmentId: 'file-err', offset: 2 })+ const missing = await runRead(tool, { attachmentId: 'file-other' })+ expect(bad.outputBody.error).toBe('range_invalid')+ expect(missing.outputBody.error).toBe('attachment_not_found')+ expect(ledger.snapshot().reads).toBe(before.reads + 2)+ expect(ledger.snapshot().spentTokens).toBeGreaterThan(before.spentTokens)+ })+})++describe('C16 finalize fence', () => {+ it('replaces post-transform growth with a trusted exhausted notice at the original offset', async () => {+ const [file] = admitted([+ rawFile('file-grow', 'grow.txt', 'text/plain', Buffer.from('growth page body ')),+ ])+ const { tool, ledger } = toolFor([file!])++ const { outputBody, finalized } = await runRead(+ tool,+ { attachmentId: 'file-grow' },+ { transform: content => content + ' '.repeat(4_000) }+ )+ expect(outputBody.kind).toBe('text')+ expect(finalized.is_error).toBe(false)+ const body = JSON.parse(finalized.rawContent as string) as Record<string, unknown>+ expect(body.kind).toBe('read_budget_exhausted')+ expect(body.resumeOffset).toBe(0)+ expect(finalized.content).not.toContain('growth page body')+ expect(finalized.emittedMessageCost).toBeLessThanOrEqual(120)+ expect(ledger.snapshot().bytesRead).toBe(0)+ })++ it('fails closed when even the trusted notice cannot fit the remaining budget', async () => {+ const [file] = admitted([+ rawFile('file-tight', 'tight.txt', 'text/plain', Buffer.from('tight page body')),+ ])+ const ledger = new AttachmentReadLedger()+ const spent = Math.floor(WINDOW_TOKENS * 0.3) - 1+ ledger.restore({ reads: 1, spentTokens: spent, bytesRead: 0 })+ const { tool } = toolFor([file!], { ledger })++ await expect(runRead(tool, { attachmentId: 'file-tight' })).rejects.toThrow(+ /cannot fit|budget/i+ )+ })++ it('finalizes exactly once without another tool execution', async () => {+ const [file] = admitted([+ rawFile('file-once', 'once.txt', 'text/plain', Buffer.from('single page body')),+ ])+ const { tool } = toolFor([file!])+ const output = await tool.execute(+ { attachmentId: 'file-once' },+ { onOutput: () => {}, measureResult }+ )+ const result: ToolResult = {+ tool_call_id: 'tc_once',+ name: TOOL_NAME,+ content: renderContent(output.content),+ is_error: output.is_error,+ emittedMessageCost: measureContent(renderContent(output.content)),+ }+ const first = tool.finalizeResult!(result, { measureContent, renderContent })+ expect(first.emittedMessageCost).toBeGreaterThan(0)+ expect(() => tool.finalizeResult!(result, { measureContent, renderContent })).toThrow(+ /pending/i+ )+ })+})++describe('C16 metadata and measurement gates', () => {+ it('emits only numbers and enums in attachmentRead metadata', async () => {+ const [file] = admitted([+ rawFile(+ 'file-meta',+ 'metadata-filename.txt',+ 'text/plain',+ Buffer.from('metadata page body')+ ),+ ])+ const { tool } = toolFor([file!])+ const { finalized } = await runRead(tool, { attachmentId: 'file-meta' })++ const meta = (finalized.metadata as Record<string, unknown>).attachmentRead as Record<+ string,+ unknown+ >+ expect(Object.keys(meta).sort()).toEqual(+ ['fileBytes', 'length', 'offset', 'origin', 'paused', 'truncated'].sort()+ )+ expect(meta.origin).toBe('current')+ expect(meta.paused).toBe(false)+ expect(JSON.stringify(meta)).not.toContain('metadata-filename.txt')+ expect(JSON.stringify(meta)).not.toContain('metadata page body')+ })++ it('fails a text read closed without measureResult while the same read succeeds with it', async () => {+ const [file] = admitted([+ rawFile('file-measure', 'measure.txt', 'text/plain', Buffer.from('measurable page body')),+ ])+ const { tool, ledger } = toolFor([file!])+ const before = ledger.snapshot()++ const output = await tool.execute({ attachmentId: 'file-measure' }, { onOutput: () => {} })+ expect(output.is_error).toBe(true)+ expect(JSON.parse(output.content).error).toBe('measurement_unavailable')+ expect(ledger.snapshot().reads).toBe(before.reads + 1)++ const witness = await runRead(tool, { attachmentId: 'file-measure' })+ expect(witness.outputBody.kind).toBe('text')+ })+})diff --git a/mcp-host/src/core/tools/__tests__/attachmentRead.test.ts b/mcp-host/src/core/tools/__tests__/attachmentRead.test.tsindex ef219fd95..a58c48ae6 100644--- a/mcp-host/src/core/tools/__tests__/attachmentRead.test.ts+++ b/mcp-host/src/core/tools/__tests__/attachmentRead.test.ts@@ -7,6 +7,8 @@ import { createHash } from 'node:crypto'import { validateIncomingAttachments } from '../../../agent/incomingAttachments'import { decodeTextContent } from '../../../internalTools/textContent'import type { IncomingMessage } from '../../../server'+import { AttachmentReadLedger } from '../../attachments/attachmentReadBudget'+import { BasicSafety } from '../../safety/safety'import type { Attachment } from '../../types'import { AttachmentReadTool } from '../attachmentRead'@@ -23,6 +25,12 @@ afterEach(() => {const READ_LIMIT = 65_536const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])+const WINDOW_TOKENS = 4_000_000+const safety = new BasicSafety()+const measureResult = (raw: string): number =>+ Math.ceil(+ Buffer.byteLength(safety.previewOutputForLlm('clerum__attachment_read', raw), 'utf8') / 4+ ) + 4function rawFile(id: string, filename: string, mimeType: string, bytes: Buffer) {return {@@ -60,11 +68,14 @@ function toolFor(attachments: Attachment[], limit = READ_LIMIT): AttachmentReadThostRef: 'host-1',attachments,}- return new AttachmentReadTool(message, limit)+ return new AttachmentReadTool(message, limit, {+ contextWindowTokens: WINDOW_TOKENS,+ ledger: new AttachmentReadLedger(),+ })}async function read(tool: AttachmentReadTool, params: Record<string, unknown>) {- const output = await tool.execute(params)+ const output = await tool.execute(params, { onOutput: () => {}, measureResult })return { output, body: JSON.parse(output.content) as Record<string, unknown> }}diff --git a/mcp-host/src/core/tools/attachmentRead.ts b/mcp-host/src/core/tools/attachmentRead.tsindex 1197aca07..0f7db1306 100644--- a/mcp-host/src/core/tools/attachmentRead.ts+++ b/mcp-host/src/core/tools/attachmentRead.ts@@ -10,8 +10,10 @@* (at most `attachmentTextReadMaxBytes` per call). A page never splits a* code point, so the text never carries U+FFFD. The caller bounds the* page, so the tool is exempt from tool-output spillover and every page- * reaches the model inline (#678); `attachmentTextReadMaxBytes` is sized- * against the context budget for that reason.+ * reaches the model inline (#678). C16 additionally fits each emitted+ * page — after JSON escaping and the sanitized wrapper — into the+ * current-turn page/turn budgets, shrinking by whole code points with+ * measurement only; the tool is never re-executed to fit.* - `reader:'none'`: a typed binary result; the bytes are never decoded.** The text-or-binary rule is the one `clerum__gfs_read` applies@@ -23,10 +25,28 @@import type { FileReferenceV1 } from '@clerum/gfs-interaction-policy'import { decodeTextContent } from '../../internalTools/textContent'import type { IncomingMessage } from '../../server'-import type { Tool, ToolTraceDescriptor } from '../interfaces'-import type { ToolOutput } from '../types'+import { AttachmentReadLedger, attachmentReadBudgets } from '../attachments/attachmentReadBudget'+import type {+ ExecutionContext,+ Tool,+ ToolEmissionContext,+ ToolTraceDescriptor,+} from '../interfaces'+import type { ToolOutput, ToolResult } from '../types'-export type AttachmentReadErrorCode = 'attachment_not_found' | 'range_invalid'+export type AttachmentReadErrorCode =+ | 'attachment_not_found'+ | 'range_invalid'+ | 'measurement_unavailable'++export interface AttachmentReadToolOptions {+ /** Positive safe integer; the ledger derives page/turn budgets from it. */+ contextWindowTokens: number+ /** Turn-owned ledger; the registry wires one per turn. */+ ledger: AttachmentReadLedger+}++export type AttachmentReadLimit = 'max_bytes' | 'page_budget' | 'turn_budget'// The result names the file by `attachmentId` (what the model passes to read// again) and `referenceId` (the FileReferenceV1 id, which pins the bytes by@@ -43,6 +63,8 @@ export type AttachmentReadResult =kind: 'text'byteRange: { offset: number; length: number }truncated: boolean+ limit?: AttachmentReadLimit+ nextOffset?: numbertext: string}| {@@ -52,17 +74,39 @@ export type AttachmentReadResult =reader: 'none'reason: 'no_reader_for_class' | 'text_decode_rejected' | 'bytes_unavailable_after_restart'}+ | {+ attachmentId: string+ referenceId?: FileReferenceV1['id']+ kind: 'read_budget_exhausted'+ resumeOffset: number+ message: string+ }const ERROR_MESSAGES: Record<AttachmentReadErrorCode, string> = {attachment_not_found: 'No file with this attachmentId is attached to the current message.',range_invalid:'offset must be an integer within the file on a character boundary, and maxBytes an integer from 1 to the per-call limit that covers at least one character.',+ measurement_unavailable:+ 'The runtime did not provide the exact output measurement this bounded reader requires.',}function isUtf8Continuation(byte: number): boolean {return (byte & 0xc0) === 0x80}+const EXHAUSTED_MESSAGE =+ 'The current-turn attachment read budget is exhausted. Reattach the file in a new message to resume at the reported offset; earlier pages are not stored for re-reading.'++/** Execute-time reservation consumed exactly once by finalizeResult. */+interface PendingEmission {+ attachmentId: string+ referenceId?: FileReferenceV1['id']+ offset: number+ fileBytes: number+ bytesRead: number+ reservedCost: number | null+}+export class AttachmentReadTool implements Tool {/** Decoded bytes and the whole-file text check, per attachmentId. */private readonly decoded = new Map<string, { bytes: Buffer; isText: boolean }>()@@ -70,8 +114,22 @@ export class AttachmentReadTool implements Tool {/** @param maxBytesPerCall the per-call ceiling and the default page. */constructor(private readonly sourceMessage: IncomingMessage,- private readonly maxBytesPerCall: number- ) {}+ private readonly maxBytesPerCall: number,+ options: AttachmentReadToolOptions+ ) {+ if (!options || typeof options !== 'object') {+ throw new Error('Attachment read options are required')+ }+ // Reuses the ledger's validation: positive safe integer or throw here,+ // before any read can execute.+ attachmentReadBudgets(options.contextWindowTokens)+ this.contextWindowTokens = options.contextWindowTokens+ this.ledger = options.ledger+ }++ private readonly contextWindowTokens: number+ private readonly ledger: AttachmentReadLedger+ private pending: PendingEmission | null = nullname(): string {return 'clerum__attachment_read'@@ -82,9 +140,10 @@ export class AttachmentReadTool implements Tool {'Read a file attached to the current message, by the attachmentId listed as attached_file in the turn context. ' +'Files with reader=text return UTF-8 text; read further pages with offset when truncated is true. ' +`A page is returned whole, up to maxBytes (default ${this.maxBytesPerCall} bytes); pass a smaller maxBytes to read less at a time. ` +- 'Files with reader=none return a binary result without content. ' +- 'A reader=text file whose bytes fail the text check also returns a binary result; ' +- 'say the file cannot be read instead of guessing its content.'+ 'Files the native UTF-8 reader cannot interpret return a binary result without content; ' ++ 'say this tool cannot read the file instead of guessing its content. ' ++ 'Reading is current-turn only: page and turn budgets bound each response, and when they are exhausted ' ++ 'you must reattach the file in a new message to continue from the reported offset.')}@@ -133,7 +192,7 @@ export class AttachmentReadTool implements Tool {return { kind: 'internal_tool', sourceRef: 'mcp-host' }}- async execute(params: Record<string, unknown>): Promise<ToolOutput> {+ async execute(params: Record<string, unknown>, context?: ExecutionContext): Promise<ToolOutput> {const start = Date.now()const attachment = this.sourceMessage.attachments?.find(candidate =>@@ -142,16 +201,28 @@ export class AttachmentReadTool implements Tool {candidate.fileReference !== undefined)const reference = attachment?.fileReference- if (!attachment || !reference) return this.error('attachment_not_found', start)+ if (!attachment || !reference) {+ if (!this.ledger.beginRead())+ return this.beginReadExhausted(attachment?.id, undefined, params)+ return this.error('attachment_not_found', start, {+ attachmentId: typeof params.attachmentId === 'string' ? params.attachmentId : '',+ offset: 0,+ fileBytes: 0,+ })+ }const identity = { attachmentId: attachment.id, referenceId: reference.id }+ if (!this.ledger.beginRead())+ return this.beginReadExhausted(identity.attachmentId, identity.referenceId, params)+// #666 R4-M2 — a cold restart persists attachment metadata but never the// inline bytes, so the tool answers honestly instead of crashing on// Buffer.from(undefined).if (typeof attachment.dataBase64 !== 'string') {return this.ok({ ...identity, kind: 'binary', reader: 'none', reason: 'bytes_unavailable_after_restart' },- start+ start,+ { identity, offset: 0, fileBytes: 0, unavailableReason: 'bytes_unavailable_after_restart' })}@@ -166,13 +237,23 @@ export class AttachmentReadTool implements Tool {maxBytes < 1 ||maxBytes > this.maxBytesPerCall) {- return this.error('range_invalid', start)+ return this.error('range_invalid', start, {+ identity,+ offset: 0,+ fileBytes: this.cachedBytes(attachment.id, attachment.dataBase64),+ })}if (reference.reader === 'none') {return this.ok({ ...identity, kind: 'binary', reader: 'none', reason: 'no_reader_for_class' },- start+ start,+ {+ identity,+ offset,+ fileBytes: this.cachedBytes(attachment.id, attachment.dataBase64),+ unavailableReason: 'no_reader_for_class',+ })}@@ -180,34 +261,153 @@ export class AttachmentReadTool implements Tool {if (!isText) {return this.ok({ ...identity, kind: 'binary', reader: 'none', reason: 'text_decode_rejected' },- start+ start,+ { identity, offset, fileBytes: bytes.length, unavailableReason: 'text_decode_rejected' })}- if (offset > bytes.length) return this.error('range_invalid', start)+ if (offset > bytes.length) {+ return this.error('range_invalid', start, { identity, offset, fileBytes: bytes.length })+ }if (offset < bytes.length && isUtf8Continuation(bytes[offset]!)) {- return this.error('range_invalid', start)+ return this.error('range_invalid', start, { identity, offset, fileBytes: bytes.length })}let end = Math.min(offset + maxBytes, bytes.length)while (end < bytes.length && isUtf8Continuation(bytes[end]!)) end -= 1- if (end === offset && offset < bytes.length) return this.error('range_invalid', start)+ if (end === offset && offset < bytes.length) {+ return this.error('range_invalid', start, { identity, offset, fileBytes: bytes.length })+ }++ const measure = context?.measureResult+ if (typeof measure !== 'function') {+ return this.error('measurement_unavailable', start, {+ identity,+ offset,+ fileBytes: bytes.length,+ })+ }++ const notice = this.exhaustedResult(identity.attachmentId, identity.referenceId, offset)+ const noticeCost = measure(JSON.stringify(notice))+ const allowance = this.ledger.pageAllowance(this.contextWindowTokens, noticeCost)+ let binding: AttachmentReadLimit | null = end < bytes.length ? 'max_bytes' : null+ const serialized = this.serializePage(bytes, identity, offset, end, end < bytes.length, binding)+ if (measure(serialized) > allowance) {+ // Binary-search the largest codepoint-aligned end that still fits. The+ // whole file is already decoded in memory; only serialization and the+ // pure measurement run per probe.+ // Advance from a boundary past the full codepoint that starts there;+ // `boundary + 1` alone lands inside a multi-byte character.+ const nextBoundary = (from: number): number => {+ let boundary = from+ while (boundary < bytes.length && isUtf8Continuation(bytes[boundary]!)) boundary += 1+ let next = boundary + 1+ while (next < bytes.length && isUtf8Continuation(bytes[next]!)) next += 1+ return next+ }+ let lo = nextBoundary(offset)+ let hi = end+ let best = -1+ while (lo <= hi) {+ const mid = Math.floor((lo + hi) / 2)+ let aligned = mid+ while (aligned > lo && isUtf8Continuation(bytes[aligned]!)) aligned -= 1+ const cost = measure(+ this.serializePage(+ bytes,+ identity,+ offset,+ aligned,+ aligned < bytes.length,+ 'page_budget'+ )+ )+ if (cost <= allowance) {+ best = aligned+ lo = nextBoundary(aligned)+ } else {+ hi = aligned - 1+ }+ }+ if (best < 0) {+ return this.ok(notice, start, {+ identity,+ offset,+ fileBytes: bytes.length,+ reservedCost: noticeCost,+ exhausted: true,+ })+ }+ end = best+ binding =+ allowance < attachmentReadBudgets(this.contextWindowTokens).pageTokens+ ? 'turn_budget'+ : 'page_budget'+ }- // A byte order mark is dropped only at the start of the file; further in,- // U+FEFF is file content and stays.- const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: offset > 0 }).decode(- bytes.subarray(offset, end)- )return this.ok(+ this.pageResult(bytes, identity, offset, end, end < bytes.length, binding),+ start,{- ...identity,- kind: 'text',- byteRange: { offset, length: end - offset },- truncated: end < bytes.length,- text,- },- start+ identity,+ offset,+ fileBytes: bytes.length,+ bytesRead: end - offset,+ reservedCost: measure(+ this.serializePage(bytes, identity, offset, end, end < bytes.length, binding)+ ),+ })}+ /**+ * Final budget fence. Runs once per executed read after safety, spillover+ * and any result transform. If the final message outgrew the reservation,+ * replace it with the trusted exhausted notice at the ORIGINAL offset; if+ * even that cannot fit, throw so publication stops without a rerun.+ */+ finalizeResult(result: ToolResult, context: ToolEmissionContext): ToolResult {+ const pending = this.pending+ if (!pending) throw new Error('AttachmentReadTool finalize called without a pending read')+ this.pending = null++ const actualCost = result.emittedMessageCost ?? context.measureContent(result.content)+ if (!Number.isSafeInteger(actualCost) || actualCost < 0) {+ throw new Error('Attachment read emission cost is not a safe non-negative integer')+ }+ const overran = pending.reservedCost !== null && actualCost > pending.reservedCost+ if (!overran && this.ledger.canEmit(this.contextWindowTokens, actualCost)) {+ this.ledger.debit(this.contextWindowTokens, actualCost, pending.bytesRead)+ return { ...result, emittedMessageCost: actualCost }+ }++ const noticeRaw = JSON.stringify(+ this.exhaustedResult(pending.attachmentId, pending.referenceId, pending.offset)+ )+ const noticeContent = context.renderContent(noticeRaw)+ const noticeCost = context.measureContent(noticeContent)+ if (!this.ledger.canEmit(this.contextWindowTokens, noticeCost)) {+ throw new Error('Attachment read notice cannot fit the remaining page/turn budget')+ }+ this.ledger.debit(this.contextWindowTokens, noticeCost, 0)+ return {+ ...result,+ content: noticeContent,+ rawContent: noticeRaw,+ is_error: false,+ emittedMessageCost: noticeCost,+ metadata: {+ attachmentRead: {+ origin: pending.offset === 0 ? 'current' : 'current_resumed',+ offset: pending.offset,+ length: 0,+ fileBytes: pending.fileBytes,+ truncated: true,+ paused: true,+ },+ },+ }+ }+// The whole-file text check runs once per attachment; each page then// decodes only its own slice.private decode(attachmentId: string, dataBase64: string): { bytes: Buffer; isText: boolean } {@@ -219,15 +419,164 @@ export class AttachmentReadTool implements Tool {return entry}- private ok(result: AttachmentReadResult, start: number): ToolOutput {- return { content: JSON.stringify(result), duration_ms: Date.now() - start, is_error: false }+ private ok(+ result: AttachmentReadResult,+ start: number,+ pending: {+ identity: { attachmentId: string; referenceId: FileReferenceV1['id'] }+ offset: number+ fileBytes: number+ bytesRead?: number+ reservedCost?: number+ exhausted?: boolean+ unavailableReason?:+ | 'no_reader_for_class'+ | 'text_decode_rejected'+ | 'bytes_unavailable_after_restart'+ }+ ): ToolOutput {+ const content = JSON.stringify(result)+ const truncated =+ result.kind === 'text' ? result.truncated : result.kind === 'read_budget_exhausted'+ const limit = result.kind === 'text' ? result.limit : undefined+ this.pending = {+ attachmentId: pending.identity.attachmentId,+ referenceId: pending.identity.referenceId,+ offset: pending.offset,+ fileBytes: pending.fileBytes,+ bytesRead: pending.bytesRead ?? 0,+ reservedCost: pending.reservedCost ?? null,+ }+ return {+ content,+ duration_ms: Date.now() - start,+ is_error: false,+ metadata: {+ attachmentRead: {+ origin: pending.offset === 0 ? 'current' : 'current_resumed',+ offset: pending.offset,+ length: pending.bytesRead ?? 0,+ fileBytes: pending.fileBytes,+ ...(truncated ? { truncated: true } : { truncated: false }),+ ...(limit ? { limit } : {}),+ ...(pending.exhausted ? { paused: true } : { paused: false }),+ ...(pending.unavailableReason ? { unavailableReason: pending.unavailableReason } : {}),+ },+ },+ }}- private error(code: AttachmentReadErrorCode, start: number): ToolOutput {+ private error(+ code: AttachmentReadErrorCode,+ start: number,+ pending: {+ identity?: { attachmentId: string; referenceId?: FileReferenceV1['id'] }+ attachmentId?: string+ offset: number+ fileBytes: number+ }+ ): ToolOutput {+ this.pending = {+ attachmentId: pending.identity?.attachmentId ?? pending.attachmentId ?? '',+ referenceId: pending.identity?.referenceId,+ offset: pending.offset,+ fileBytes: pending.fileBytes,+ bytesRead: 0,+ reservedCost: null,+ }return {content: JSON.stringify({ error: code, message: ERROR_MESSAGES[code] }),duration_ms: Date.now() - start,is_error: true,}}++ private beginReadExhausted(+ attachmentId: string | undefined,+ referenceId: FileReferenceV1['id'] | undefined,+ params: Record<string, unknown>+ ): ToolOutput {+ const offset =+ typeof params.offset === 'number' && Number.isSafeInteger(params.offset) && params.offset >= 0+ ? params.offset+ : 0+ const result = this.exhaustedResult(attachmentId ?? '', referenceId, offset)+ this.pending = {+ attachmentId: attachmentId ?? '',+ referenceId,+ offset,+ fileBytes: 0,+ bytesRead: 0,+ reservedCost: null,+ }+ return {+ content: JSON.stringify(result),+ duration_ms: 0,+ is_error: false,+ metadata: {+ attachmentRead: {+ origin: offset === 0 ? 'current' : 'current_resumed',+ offset,+ length: 0,+ fileBytes: 0,+ truncated: true,+ paused: true,+ },+ },+ }+ }++ private exhaustedResult(+ attachmentId: string,+ referenceId: FileReferenceV1['id'] | undefined,+ resumeOffset: number+ ): AttachmentReadResult {+ return {+ attachmentId,+ ...(referenceId !== undefined ? { referenceId } : {}),+ kind: 'read_budget_exhausted',+ resumeOffset,+ message: EXHAUSTED_MESSAGE,+ }+ }++ private pageResult(+ bytes: Buffer,+ identity: { attachmentId: string; referenceId: FileReferenceV1['id'] },+ offset: number,+ end: number,+ truncated: boolean,+ limit: AttachmentReadLimit | null+ ): AttachmentReadResult {+ // A byte order mark is dropped only at the start of the file; further in,+ // U+FEFF is file content and stays.+ const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: offset > 0 }).decode(+ bytes.subarray(offset, end)+ )+ return {+ attachmentId: identity.attachmentId,+ referenceId: identity.referenceId,+ kind: 'text',+ byteRange: { offset, length: end - offset },+ truncated,+ ...(truncated && limit ? { limit } : {}),+ ...(truncated ? { nextOffset: end } : {}),+ text,+ }+ }++ private serializePage(+ bytes: Buffer,+ identity: { attachmentId: string; referenceId: FileReferenceV1['id'] },+ offset: number,+ end: number,+ truncated: boolean,+ limit: AttachmentReadLimit | null+ ): string {+ return JSON.stringify(this.pageResult(bytes, identity, offset, end, truncated, limit))+ }++ private cachedBytes(attachmentId: string, dataBase64: string): number {+ return this.decode(attachmentId, dataBase64).bytes.length+ }}diff --git a/mcp-host/src/core/types.ts b/mcp-host/src/core/types.tsindex 191244f45..4cbf766e8 100644--- a/mcp-host/src/core/types.ts+++ b/mcp-host/src/core/types.ts@@ -310,6 +310,8 @@ export interface ToolResult {is_error: booleanattachments?: Attachment[]metadata?: Record<string, unknown>+ /** Token estimate of the final model-visible message; not part of its content. */+ emittedMessageCost?: number/*** Pre-sanitization content for user-facing output preview.* WARNING: bypasses the XML safety wrapper. Do NOT use in LLM messages.