Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(mcp-host): take the refusal messages from the real contracts

Alfredo Lopez08c58333643f
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index b4fc20acb..601af23c9 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.469",
+ "version": "0.1.470",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.469",
+ "version": "0.1.470",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index c6c798718..a9d4f6471 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.469",
+ "version": "0.1.470",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts b/mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts
index e92ccca4d..cf3a7968e 100644
--- a/mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts
+++ b/mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts
@@ -1,14 +1,23 @@
import { describe, expect, it } from 'vitest'
-import { LIMITS as GROK_LIMITS, GROK_VISUAL_LIMITS } from '@clerum/grok-provider-attempt-contract'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import {
+ LIMITS as GROK_LIMITS,
+ GROK_VISUAL_LIMITS,
+ hashCanonicalGrokRequest,
+} from '@clerum/grok-provider-attempt-contract'
import {
LIMITS as CODEX_LIMITS,
VISUAL_LIMITS as CODEX_VISUAL_LIMITS,
+ hashCanonicalCodexRequest,
} from '@clerum/llm-provider-attempt-contract'
import {
attachmentBudgetRefusalMessageFor,
buildAttachmentBudgetRefusals,
} from '../attachmentBudgetRefusal'
import { attachmentBudgetRefusalMessage } from '../codexSubscription'
+import { PNG_OVER_DIMENSION_BASE64, pngOfDecodedBytesBase64 } from './codexImageFixtures'
+import { grokPngOfDecodedBytesBase64 } from './grokImageFixtures'
// #784: the R9-11 refusal table, built from a provider's own limits.
@@ -25,14 +34,111 @@ const CODEX_TABLE = buildAttachmentBudgetRefusals({
maxImagePixels: CODEX_VISUAL_LIMITS.maxImagePixels,
})
-// The contract messages, byte-identical across both contracts (G1).
-const PER_IMAGE = (bytes: number) =>
- `messages[0].contentParts[1]: image exceeds ${bytes} decoded bytes`
-const DIMENSION = (px: number) => `messages[0].contentParts[1]: image dimension exceeds ${px}`
-const PIXELS = (count: number) => `messages[0].contentParts[1]: image pixel count exceeds ${count}`
-const TOTAL = (bytes: number) => `request exceeds ${bytes} total image bytes`
-const VISUAL_BODY = 'request exceeds maxVisualRequestBodyBytes'
-const OUTSIDE_IMAGE_DATA = 'request exceeds maxRequestBodyBytes outside image data'
+// Every refusal message below is produced by the contract itself, from its
+// valid V2 PNG fixture with one field changed, so a reworded contract message
+// breaks this suite instead of leaving the table matching a stale copy.
+type TextPart = { type: 'text'; text: string }
+type ImagePart = { type: 'image'; data: string }
+type FixtureRequest = {
+ schemaVersion: string
+ messages: Array<{ role: string; content: string; contentParts: Array<TextPart | ImagePart> }>
+}
+
+const fixtureRequest = (contractPackage: string): FixtureRequest =>
+ JSON.parse(
+ readFileSync(
+ join(__dirname, `../../../../packages/${contractPackage}/fixtures/visual-requests.json`),
+ 'utf8'
+ )
+ ).png
+
+const refusalFrom =
+ (contractPackage: string, hash: (raw: unknown) => { ok: boolean; message?: string }) =>
+ (change: (request: FixtureRequest) => void): string => {
+ const request = fixtureRequest(contractPackage)
+ change(request)
+ const result = hash(request)
+ if (result.ok || typeof result.message !== 'string') {
+ throw new Error(`the ${contractPackage} contract accepted a request built to be refused`)
+ }
+ return result.message
+ }
+
+const grokRefusal = refusalFrom('grok-provider-attempt-contract', hashCanonicalGrokRequest)
+const codexRefusal = refusalFrom('llm-provider-attempt-contract', hashCanonicalCodexRequest)
+
+const imageParts = (request: FixtureRequest): ImagePart[] =>
+ request.messages[0].contentParts.filter((part): part is ImagePart => part.type === 'image')
+
+/** Replaces the fixture image with one image per entry of `data`. */
+const withImages =
+ (...data: string[]) =>
+ (request: FixtureRequest): void => {
+ const [image] = imageParts(request)
+ const text = request.messages[0].contentParts.filter(part => part.type === 'text')
+ request.messages[0].contentParts = [...text, ...data.map(entry => ({ ...image, data: entry }))]
+ }
+
+/** Pads the single text part, and `content` with it, by `bytes` ASCII bytes. */
+const withTextPadding =
+ (bytes: number) =>
+ (request: FixtureRequest): void => {
+ const message = request.messages[0]
+ const text = message.contentParts.find((part): part is TextPart => part.type === 'text')
+ if (!text) throw new Error('the fixture has no text part')
+ text.text += 'x'.repeat(bytes)
+ message.content = text.text
+ }
+
+/** Pads the image data. The whole-body ceiling is measured before any image is decoded. */
+const withImageDataPadding =
+ (bytes: number) =>
+ (request: FixtureRequest): void => {
+ imageParts(request)[0].data += 'A'.repeat(bytes)
+ }
+
+const GROK = {
+ perImage: grokRefusal(
+ withImages(grokPngOfDecodedBytesBase64(GROK_VISUAL_LIMITS.maxImageBytes + 1))
+ ),
+ total: grokRefusal(
+ withImages(
+ grokPngOfDecodedBytesBase64(GROK_VISUAL_LIMITS.maxTotalImageBytes / 2 + 1),
+ grokPngOfDecodedBytesBase64(GROK_VISUAL_LIMITS.maxTotalImageBytes / 2 + 1)
+ )
+ ),
+ visualBody: grokRefusal(withImageDataPadding(GROK_LIMITS.maxVisualRequestBodyBytes)),
+ outsideImageData: grokRefusal(withTextPadding(GROK_LIMITS.maxRequestBodyBytes)),
+ v1Body: grokRefusal(request => {
+ request.schemaVersion = 'grok-completion-request.v1'
+ request.messages = [
+ { role: 'user', content: 'x'.repeat(GROK_LIMITS.maxRequestBodyBytes) },
+ ] as FixtureRequest['messages']
+ }),
+}
+
+const CODEX = {
+ perImage: codexRefusal(
+ withImages(pngOfDecodedBytesBase64(CODEX_VISUAL_LIMITS.maxImageBytes + 1))
+ ),
+ dimension: codexRefusal(withImages(PNG_OVER_DIMENSION_BASE64)),
+ total: codexRefusal(
+ withImages(
+ pngOfDecodedBytesBase64(CODEX_VISUAL_LIMITS.maxTotalImageBytes / 2 + 1),
+ pngOfDecodedBytesBase64(CODEX_VISUAL_LIMITS.maxTotalImageBytes / 2 + 1)
+ )
+ ),
+ visualBody: codexRefusal(withImageDataPadding(CODEX_LIMITS.maxVisualRequestBodyBytes)),
+ outsideImageData: codexRefusal(withTextPadding(CODEX_LIMITS.maxRequestBodyBytes)),
+}
+
+// The one message no request can produce today: Codex `maxImagePixels` equals
+// `maxImageDimension` squared and the dimension check runs first. It is written
+// in the shape of the dimension message the contract does produce.
+const CODEX_PIXELS = CODEX.dimension.replace(
+ `image dimension exceeds ${CODEX_VISUAL_LIMITS.maxImageDimension}`,
+ `image pixel count exceeds ${CODEX_VISUAL_LIMITS.maxImagePixels}`
+)
describe('attachmentBudgetRefusal (#784)', () => {
it('names the Grok limits in the per-image, total and whole-body sentences', () => {
@@ -40,23 +146,13 @@ describe('attachmentBudgetRefusal (#784)', () => {
expect(GROK_VISUAL_LIMITS.maxTotalImageBytes).toBe(20971520)
expect(GROK_LIMITS.maxVisualRequestBodyBytes).toBe(36700160)
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- PER_IMAGE(GROK_VISUAL_LIMITS.maxImageBytes),
- true
- )
- ).toBe('An attached image is too large: it exceeds 20 MiB. Reduce its size and send it again.')
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- TOTAL(GROK_VISUAL_LIMITS.maxTotalImageBytes),
- true
- )
- ).toBe(
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.perImage, true)).toBe(
+ 'An attached image is too large: it exceeds 20 MiB. Reduce its size and send it again.'
+ )
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.total, true)).toBe(
'The attached images are too large together: they exceed 20 MiB in total. Send fewer or smaller images.'
)
- expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, VISUAL_BODY, true)).toBe(
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.visualBody, true)).toBe(
'The message and its attached images are too large together: they exceed 35 MiB. Send fewer or smaller images.'
)
})
@@ -64,79 +160,43 @@ describe('attachmentBudgetRefusal (#784)', () => {
it('has no dimension or pixel rows for Grok, which has no such limit', () => {
// Witness: the same messages are attachment refusals in a table built with
// the limits, so the Grok `undefined` is the absence of the rows.
- expect(
- attachmentBudgetRefusalMessageFor(
- CODEX_TABLE,
- DIMENSION(CODEX_VISUAL_LIMITS.maxImageDimension),
- true
- )
- ).toBe(
+ expect(attachmentBudgetRefusalMessageFor(CODEX_TABLE, CODEX.dimension, true)).toBe(
`An attached image is too large: its width or height exceeds ${CODEX_VISUAL_LIMITS.maxImageDimension} pixels. Resize it and send it again.`
)
- expect(
- attachmentBudgetRefusalMessageFor(
- CODEX_TABLE,
- PIXELS(CODEX_VISUAL_LIMITS.maxImagePixels),
- true
- )
- ).toBe(
+ expect(attachmentBudgetRefusalMessageFor(CODEX_TABLE, CODEX_PIXELS, true)).toBe(
`An attached image is too large: it has more than ${CODEX_VISUAL_LIMITS.maxImagePixels.toLocaleString('en-US')} pixels. Resize it and send it again.`
)
expect(GROK_TABLE).toHaveLength(3)
expect(CODEX_TABLE).toHaveLength(5)
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- DIMENSION(CODEX_VISUAL_LIMITS.maxImageDimension),
- true
- )
- ).toBeUndefined()
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- PIXELS(CODEX_VISUAL_LIMITS.maxImagePixels),
- true
- )
- ).toBeUndefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, CODEX.dimension, true)).toBeUndefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, CODEX_PIXELS, true)).toBeUndefined()
})
it('blames the whole-body ceiling on an attachment only when the request carries an image', () => {
- expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, VISUAL_BODY, true)).toBeDefined()
- expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, VISUAL_BODY, false)).toBeUndefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.visualBody, true)).toBeDefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.visualBody, false)).toBeUndefined()
// The per-image row does not depend on the flag: only an image can break it.
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- PER_IMAGE(GROK_VISUAL_LIMITS.maxImageBytes),
- false
- )
- ).toBeDefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.perImage, false)).toBeDefined()
})
it('leaves conversation-volume refusals to the context-length taxonomy', () => {
- expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, OUTSIDE_IMAGE_DATA, true)).toBeUndefined()
expect(
- attachmentBudgetRefusalMessageFor(GROK_TABLE, 'request exceeds maxRequestBodyBytes', true)
+ attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.outsideImageData, true)
).toBeUndefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.v1Body, true)).toBeUndefined()
// Witness: the same table maps an image refusal in the same call shape.
- expect(
- attachmentBudgetRefusalMessageFor(
- GROK_TABLE,
- TOTAL(GROK_VISUAL_LIMITS.maxTotalImageBytes),
- true
- )
- ).toBeDefined()
+ expect(attachmentBudgetRefusalMessageFor(GROK_TABLE, GROK.total, true)).toBeDefined()
})
it('keeps the Codex export equal to the table built from the Codex limits', () => {
const corpus: Array<[string, boolean]> = [
- [PER_IMAGE(CODEX_VISUAL_LIMITS.maxImageBytes), true],
- [DIMENSION(CODEX_VISUAL_LIMITS.maxImageDimension), true],
- [PIXELS(CODEX_VISUAL_LIMITS.maxImagePixels), true],
- [TOTAL(CODEX_VISUAL_LIMITS.maxTotalImageBytes), true],
- [VISUAL_BODY, true],
- [VISUAL_BODY, false],
- [OUTSIDE_IMAGE_DATA, true],
+ [CODEX.perImage, true],
+ [CODEX.dimension, true],
+ [CODEX_PIXELS, true],
+ [CODEX.total, true],
+ [CODEX.visualBody, true],
+ [CODEX.visualBody, false],
+ [CODEX.outsideImageData, true],
]
let mapped = 0
for (const [message, carriesImage] of corpus) {