Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): drop leftover visual payloads on deny, suspend, and failover

Alfredo Lopez8dc485dc7392
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 9f9d220a9..fd8e4ee80 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.342",
+ "version": "0.1.343",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.342",
+ "version": "0.1.343",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 74be27fa3..3ba267817 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.342",
+ "version": "0.1.343",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/agent/__tests__/taskExecutionBudget.test.ts b/mcp-host/src/agent/__tests__/taskExecutionBudget.test.ts
index 1947d877e..912068e1a 100644
--- a/mcp-host/src/agent/__tests__/taskExecutionBudget.test.ts
+++ b/mcp-host/src/agent/__tests__/taskExecutionBudget.test.ts
@@ -1,4 +1,5 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
+import { VISUAL_INPUT_LIMITS } from '../../visualInput/policy'
import {
TaskExecutionBudget,
TaskLimitError,
@@ -51,6 +52,13 @@ describe('TaskExecutionBudget', () => {
{},
{ elapsedActiveMs: -1, iterationsUsed: 0 },
{ elapsedActiveMs: 0, iterationsUsed: NaN },
+ {
+ elapsedActiveMs: 0,
+ iterationsUsed: 0,
+ durationMs: 1,
+ maxIterations: 1,
+ visualReadBytes: VISUAL_INPUT_LIMITS.readBytesPerTurn + 1,
+ },
])('rejects invalid durable accounting %s', value => {
expect(() => parseTaskExecutionBudget(value)).toThrow('Invalid task execution budget')
})
diff --git a/mcp-host/src/core/adapters/__tests__/llmPortAdapter.test.ts b/mcp-host/src/core/adapters/__tests__/llmPortAdapter.test.ts
index 81850ffff..9836b5a61 100644
--- a/mcp-host/src/core/adapters/__tests__/llmPortAdapter.test.ts
+++ b/mcp-host/src/core/adapters/__tests__/llmPortAdapter.test.ts
@@ -678,3 +678,74 @@ describe('Cross-provider ToolCall normalization', () => {
})
})
})
+
+describe('LlmPortAdapter image destination degrade', () => {
+ const imageMessage = {
+ role: 'user' as const,
+ content: 'image',
+ contentParts: [
+ {
+ type: 'image' as const,
+ mimeType: 'image/png' as const,
+ data: 'pixel-bytes',
+ source: {
+ kind: 'gfs' as const,
+ drive: 'main',
+ resourceId: 'a'.repeat(32),
+ gfsUri: `gfs://main/${'a'.repeat(32)}`,
+ version: 1,
+ name: 'pic.png',
+ },
+ },
+ ],
+ }
+
+ it('strips unverified image parts and continues the turn', async () => {
+ const completeSingleTurnWithTools = vi.fn().mockResolvedValue({
+ content: 'ok',
+ tool_calls: [],
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: FinishReason.Stop,
+ })
+ const adapter = new LlmPortAdapter(
+ {
+ completeSingleTurn: vi.fn(),
+ completeSingleTurnWithTools,
+ getProviderType: () => 'openai' as const,
+ getImageInputCapability: async () => ({ status: 'unknown' as const }),
+ classifyError: vi.fn(),
+ },
+ 'unknown-model',
+ 'openai'
+ )
+ const messages = [structuredClone(imageMessage)]
+ const result = await adapter.completeWithTools({ messages, tools: [] })
+ expect(result.content).toBe('ok')
+ expect(completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
+ const sent = completeSingleTurnWithTools.mock.calls[0][0]
+ expect(
+ sent.flatMap((m: { contentParts?: { type: string }[] }) => m.contentParts ?? [])
+ ).not.toEqual(expect.arrayContaining([expect.objectContaining({ type: 'image' })]))
+ expect(JSON.stringify(sent)).toContain('image_input_not_verified_for_selected_model')
+ expect(JSON.stringify(sent)).not.toContain('pixel-bytes')
+ })
+
+ it('still rejects image parts that left the user visual message', async () => {
+ const adapter = new LlmPortAdapter(
+ {
+ completeSingleTurn: vi.fn(),
+ completeSingleTurnWithTools: vi.fn(),
+ getProviderType: () => 'openai' as const,
+ classifyError: vi.fn(),
+ },
+ 'gpt-4o',
+ 'openai'
+ )
+ await expect(
+ adapter.completeWithTools({
+ messages: [{ ...imageMessage, role: 'system' }],
+ tools: [],
+ })
+ ).rejects.toThrow('Image input must remain in its user visual message')
+ })
+})
diff --git a/mcp-host/src/core/adapters/llmPortAdapter.ts b/mcp-host/src/core/adapters/llmPortAdapter.ts
index d87e231ca..0bc2829e8 100644
--- a/mcp-host/src/core/adapters/llmPortAdapter.ts
+++ b/mcp-host/src/core/adapters/llmPortAdapter.ts
@@ -11,7 +11,10 @@ import {
type VisualInputBudget,
VisualInputError,
} from '../../visualInput/policy'
-import { assertVisualRequestFits, hasGfsImageInput } from '../../visualInput/requestPolicy'
+import {
+ assertVisualRequestFits,
+ degradeUnverifiedImageInput,
+} from '../../visualInput/requestPolicy'
import type { SessionTokenUsage } from '../conversation/conversationStore'
import { LlmError, LlmErrorCode } from '../errors'
import { LlmPort } from '../interfaces'
@@ -138,8 +141,7 @@ export class LlmPortAdapter implements LlmPort {
const hasImages = request.messages.some(message =>
message.contentParts?.some(part => part.type === 'image')
)
- if (!hasGfsImageInput(request.messages) && !(this.visualBudget?.hasEncodedImages && hasImages))
- return false
+ if (!hasImages) return false
if (
request.messages.some(
message =>
@@ -160,12 +162,8 @@ export class LlmPortAdapter implements LlmPort {
capability.provider !== this.providerName ||
capability.model !== this.model
) {
- throw new LlmError(
- 'Image input is not verified for the selected model and request method. Read the resource again with a supported model.',
- this.providerName,
- LlmErrorCode.ApiCallFailed,
- false
- )
+ degradeUnverifiedImageInput(request.messages)
+ return false
}
assertVisualRequestFits(request.messages, request, true)
return true
diff --git a/mcp-host/src/core/conversation/conversation.ts b/mcp-host/src/core/conversation/conversation.ts
index 528d47bf5..3ba8b7f29 100644
--- a/mcp-host/src/core/conversation/conversation.ts
+++ b/mcp-host/src/core/conversation/conversation.ts
@@ -1,4 +1,5 @@
import { parseSessionKey } from '../../session/types'
+import { projectGfsApproval } from '../../visualInput/suspension'
import { ConversationError, ConversationErrorCode } from '../errors'
import { isMcpToolName } from '../extensions/mcpApprovalGateController'
import {
@@ -511,11 +512,12 @@ export class ConversationManager {
const prevApproval = conversation.pending_approval
const prevApprovalTraceContext = approval.traceContext
approval.traceContext = conversation.traceContext ?? null
+ const projected = projectGfsApproval(approval)
conversation.state = ConversationState.AwaitingApproval
- conversation.pending_approval = approval
+ conversation.pending_approval = projected
conversation.updated_at = new Date()
try {
- await this.store.persistSuspend(conversation, approval)
+ await this.store.persistSuspend(conversation, projected)
} catch (err) {
// Under the sqlite/dual store the durable write can reject (worker
// timeout, SQLITE_BUSY, worker exit). Roll back the in-RAM mutation so
diff --git a/mcp-host/src/core/conversation/persistence/__tests__/sqliteConversationStore.test.ts b/mcp-host/src/core/conversation/persistence/__tests__/sqliteConversationStore.test.ts
index f7cbb7576..3168c3e94 100644
--- a/mcp-host/src/core/conversation/persistence/__tests__/sqliteConversationStore.test.ts
+++ b/mcp-host/src/core/conversation/persistence/__tests__/sqliteConversationStore.test.ts
@@ -94,6 +94,8 @@ describe('SqliteConversationStore — basic round-trip', () => {
expect(restored.tool_call_id).toBe('pending-action')
expect(restored.context_snapshot.slice(0, 2)).toEqual(approval.context_snapshot.slice(0, 2))
expect(restored.completed_results![0].attachments).toBeUndefined()
+ expect(JSON.stringify(conv.pending_approval)).not.toContain(payload)
+ expect(conv.pending_approval?.completed_results?.[0].attachments).toBeUndefined()
} finally {
await handle.shutdown()
}
diff --git a/mcp-host/src/core/conversation/persistence/sqliteConversationStore.ts b/mcp-host/src/core/conversation/persistence/sqliteConversationStore.ts
index e1cf5ab0e..ffca92441 100644
--- a/mcp-host/src/core/conversation/persistence/sqliteConversationStore.ts
+++ b/mcp-host/src/core/conversation/persistence/sqliteConversationStore.ts
@@ -1014,6 +1014,9 @@ export class SqliteConversationStore implements ConversationStore {
async persistSuspend(conv: Conversation, approval: PendingApproval): Promise<void> {
approval = projectGfsApproval(approval)
+ if (conv.pending_approval?.request_id === approval.request_id) {
+ conv.pending_approval = approval
+ }
const sessionKey = this.sessionKeyById.get(conv.id)
if (!sessionKey) return
if (!conv.activeTaskId) {
diff --git a/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts b/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
index 9919bb320..711acfbb2 100644
--- a/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
+++ b/mcp-host/src/core/orchestration/__tests__/toolUseLoop.test.ts
@@ -2241,6 +2241,50 @@ describe('executeSingleTool — progress watcher', () => {
})
})
+describe('executeSingleTool — failed visual leftovers', () => {
+ it('does not observe or keep attachments from a failed tool result', async () => {
+ const observeExternalImage = vi.fn()
+ const tool = {
+ name: () => 'screenshots',
+ execute: vi.fn(async () => ({
+ content: 'failed',
+ is_error: true,
+ duration_ms: 1,
+ attachments: [
+ {
+ id: 'shot',
+ kind: 'image',
+ mimeType: 'image/png',
+ encoding: 'base64',
+ dataBase64: 'failed-image',
+ },
+ ],
+ })),
+ requiresSanitization: () => false,
+ }
+ const result = await executeSingleTool({ id: 'call-1', name: 'screenshots', arguments: {} }, {
+ toolRegistry: { get: () => tool },
+ toolOutputProcessor: {
+ beforeExecution: () => ({ is_valid: true, errors: [] }),
+ afterExecution: (_n: string, out: { content: string }) => out.content,
+ },
+ safety: {
+ sanitizeOutput: (_n: string, output: string) => ({
+ content: output,
+ was_modified: false,
+ warnings: [],
+ }),
+ },
+ events: { emit: () => {} },
+ toolTimeout: 60_000,
+ visualInput: { budget: { observeExternalImage } },
+ } as never)
+ expect(result.is_error).toBe(true)
+ expect(result.attachments).toBeUndefined()
+ expect(observeExternalImage).not.toHaveBeenCalled()
+ })
+})
+
describe('tool presentation failure boundary', () => {
it('does not disclose the full local catalog or call reasoning when presentation fails', async () => {
const reasoning = createMockReasoning([{ type: 'text', content: 'should not run' }])
diff --git a/mcp-host/src/core/orchestration/toolUseLoopMessages.ts b/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
index ee636ed70..aaa296c82 100644
--- a/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
+++ b/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
@@ -140,6 +140,8 @@ export function appendToolResults(
resource: att.visualSource,
})
tr.rawContent = tr.content
+ tr.attachments = tr.attachments?.filter(existing => existing !== att)
+ if (!tr.attachments?.length) delete tr.attachments
continue
}
}
diff --git a/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts b/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts
index 322432fd6..2dd4ebb98 100644
--- a/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts
+++ b/mcp-host/src/core/orchestration/toolUseLoopSingleTool.ts
@@ -124,9 +124,12 @@ export async function executeSingleTool(
'Tool execution finished'
)
- for (const attachment of output.attachments ?? []) {
- if (attachment.kind === 'image' && !attachment.visualSource)
- config.visualInput?.budget.observeExternalImage(attachment.dataBase64)
+ const attachments = output.is_error ? undefined : output.attachments
+ if (!output.is_error) {
+ for (const attachment of attachments ?? []) {
+ if (attachment.kind === 'image' && !attachment.visualSource)
+ config.visualInput?.budget.observeExternalImage(attachment.dataBase64)
+ }
}
let wrappedContent: string
if (tool.requiresSanitization()) {
@@ -214,7 +217,7 @@ export async function executeSingleTool(
name: call.name,
content: finalContent,
is_error: output.is_error,
- attachments: output.attachments,
+ attachments,
metadata: output.metadata,
rawContent: output.content,
spillover_ref: spilloverRef,
diff --git a/mcp-host/src/internalTools/gfs.test.ts b/mcp-host/src/internalTools/gfs.test.ts
index 9255e8243..3c596796c 100644
--- a/mcp-host/src/internalTools/gfs.test.ts
+++ b/mcp-host/src/internalTools/gfs.test.ts
@@ -1,4 +1,6 @@
import { describe, expect, it, vi } from 'vitest'
+import { createCanvas } from '@napi-rs/canvas'
+import { VisualInputBudget } from '../visualInput/policy'
import {
type GfscReadClient,
type GfscWriteClient,
@@ -95,6 +97,91 @@ describe('buildGfsReadTools', () => {
expect(r.error).not.toContain('blob/key')
})
+ it('returns typed image input when the destination supports vision', async () => {
+ const canvas = createCanvas(2, 2)
+ canvas.getContext('2d').fillRect(0, 0, 2, 2)
+ const bytes = canvas.toBuffer('image/png')
+ const source = {
+ kind: 'gfs' as const,
+ drive: 'main',
+ resourceId: 'a'.repeat(32),
+ gfsUri: `gfs://main/${'a'.repeat(32)}`,
+ version: 2,
+ name: 'pic.png',
+ }
+ const reservation = { release: vi.fn() }
+ const c = client({
+ read: vi.fn(async () => ({ source, bytes, reservation })),
+ })
+ const r = await toolMap(c)
+ .get('clerum__gfs_read')!
+ .execute({ drive: 'main', resourceId: source.resourceId }, '', {
+ visualInput: {
+ budget: new VisualInputBudget(),
+ resolveCapability: async () => ({
+ status: 'supported',
+ provider: 'openai',
+ model: 'gpt-4.1',
+ evidence: 'unit',
+ }),
+ },
+ })
+ expect(r.success).toBe(true)
+ expect(JSON.parse(r.content as string).delivery).toBe('image_input')
+ expect(r.images?.[0]).toMatchObject({ mimeType: 'image/png', source })
+ expect(reservation.release).toHaveBeenCalledOnce()
+ })
+
+ it.each(['<svg/>', '<svg />', '<SVG/>', '\uFEFF<svg/>'])(
+ 'returns a reference for self-closing SVG %j',
+ async text => {
+ const c = client({
+ read: vi.fn(async () => ({
+ source: {
+ kind: 'gfs' as const,
+ drive: 'main',
+ resourceId: 'a'.repeat(32),
+ gfsUri: `gfs://main/${'a'.repeat(32)}`,
+ version: 2,
+ name: 'icon.svg',
+ },
+ bytes: Buffer.from(text),
+ reservation: { release: vi.fn() },
+ })),
+ })
+ const r = await toolMap(c)
+ .get('clerum__gfs_read')!
+ .execute({ drive: 'main', resourceId: 'abc' }, '')
+ expect(r.success).toBe(true)
+ expect(JSON.parse(r.content as string)).toMatchObject({
+ delivery: 'reference_only',
+ reason: 'svg_visual_input_not_supported',
+ })
+ }
+ )
+
+ it('strips a UTF-8 BOM from otherwise valid text', async () => {
+ const c = client({
+ read: vi.fn(async () => ({
+ source: {
+ kind: 'gfs' as const,
+ drive: 'main',
+ resourceId: 'a'.repeat(32),
+ gfsUri: `gfs://main/${'a'.repeat(32)}`,
+ version: 2,
+ name: 'note.txt',
+ },
+ bytes: Buffer.from('\uFEFFhello'),
+ reservation: { release: vi.fn() },
+ })),
+ })
+ const r = await toolMap(c)
+ .get('clerum__gfs_read')!
+ .execute({ drive: 'main', resourceId: 'abc' }, '')
+ expect(r.success).toBe(true)
+ expect(r.content).toBe('hello')
+ })
+
it('strips a non-gfsc-shaped read error down to the generic label', async () => {
const c = client({
read: vi.fn(async () => {
diff --git a/mcp-host/src/internalTools/gfs.ts b/mcp-host/src/internalTools/gfs.ts
index b46961505..b89708067 100644
--- a/mcp-host/src/internalTools/gfs.ts
+++ b/mcp-host/src/internalTools/gfs.ts
@@ -71,7 +71,7 @@ function isSvgText(text: string): boolean {
const suffixLength = remaining.startsWith('<?') ? 2 : 3
remaining = remaining.slice(end + suffixLength).trimStart()
}
- return /^<svg[\s>]/i.test(remaining) || /^<!DOCTYPE\s+svg[\s>]/i.test(remaining)
+ return /^<svg[\s>/]/i.test(remaining) || /^<!DOCTYPE\s+svg[\s>]/i.test(remaining)
}
// Locally-authored argument-validation messages carry no server data and must
// reach the model verbatim so the agent can correct its call.
@@ -181,7 +181,12 @@ export function buildGfsReadTools(client: GfscReadClient): InternalToolDefinitio
if (isNonTextFormat(file.bytes)) return fileReference(file, 'unsupported_binary_format')
let text: string
try {
- text = new TextDecoder('utf-8', { fatal: true }).decode(file.bytes)
+ // Same fileBytes cap as images (G0). TextDecoder already drops a
+ // UTF-8 BOM; strip again so classification and the model see the
+ // same contract.
+ text = new TextDecoder('utf-8', { fatal: true })
+ .decode(file.bytes)
+ .replace(/^\uFEFF/, '')
} catch {
return fileReference(file, 'unsupported_binary_format')
}
diff --git a/mcp-host/src/internalTools/gfsClient.test.ts b/mcp-host/src/internalTools/gfsClient.test.ts
index 37dc787cc..45d0cb6a5 100644
--- a/mcp-host/src/internalTools/gfsClient.test.ts
+++ b/mcp-host/src/internalTools/gfsClient.test.ts
@@ -67,6 +67,7 @@ describe('gfs runtime gfsc client', () => {
expect(fetchFn.mock.calls[0]?.[1]?.headers).toMatchObject({
authorization: 'Bearer gfs-access',
})
+ expect(fetchFn.mock.calls[0]?.[1]).toMatchObject({ redirect: 'error' })
})
it('routes writes to the gfsc writer service with conditional body', async () => {
@@ -462,7 +463,7 @@ describe('GFS binary content snapshots', () => {
})
expect(cancel).toHaveBeenCalled()
expect(budget.residentBytes).toBe(0)
- expect(budget.readBytes).toBe(8)
+ expect(budget.readBytes).toBe(3)
})
it('rejects short content and unsolicited partial responses', async () => {
diff --git a/mcp-host/src/internalTools/gfsClient.ts b/mcp-host/src/internalTools/gfsClient.ts
index 6e9925212..044f43f0d 100644
--- a/mcp-host/src/internalTools/gfsClient.ts
+++ b/mcp-host/src/internalTools/gfsClient.ts
@@ -91,6 +91,7 @@ export function createGfscClient(env: GfsRuntimeEnv): GfscWriteClient {
): Promise<Response> {
return fetchFn(`${baseUrl}${path}`, {
...init,
+ redirect: 'error',
headers: {
[AUTH_HEADER]: `${AUTH_SCHEME} ${await accessValue()}`,
...(init.headers ?? {}),
diff --git a/mcp-host/src/internalTools/gfsContentRead.ts b/mcp-host/src/internalTools/gfsContentRead.ts
index 1d282d920..fdf4be657 100644
--- a/mcp-host/src/internalTools/gfsContentRead.ts
+++ b/mcp-host/src/internalTools/gfsContentRead.ts
@@ -34,8 +34,8 @@ async function collect(
const next = await reader.read()
if (signal.aborted) throw new VisualInputError('cancelled')
if (next.done) break
- onChunk?.(next.value.byteLength)
if (next.value.byteLength > maximum - size) throw new VisualInputError('limit_exceeded')
+ onChunk?.(next.value.byteLength)
size += next.value.byteLength
chunks.push(next.value)
}
diff --git a/mcp-host/src/internalTools/gfsImageInput.integration.test.ts b/mcp-host/src/internalTools/gfsImageInput.integration.test.ts
index 4ced2cd7f..3dfa43855 100644
--- a/mcp-host/src/internalTools/gfsImageInput.integration.test.ts
+++ b/mcp-host/src/internalTools/gfsImageInput.integration.test.ts
@@ -401,8 +401,20 @@ describe('GFS bytes to actual provider request', () => {
: part
),
}))
+ const completeSingleTurnWithTools = vi.fn().mockResolvedValue({
+ content: 'degraded',
+ tool_calls: [],
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: 'stop',
+ })
const fallback = new LlmPortAdapter(
- new OpenAIProvider('integration-only', 'unknown-model'),
+ {
+ completeSingleTurn: vi.fn(),
+ completeSingleTurnWithTools,
+ getProviderType: () => 'openai' as const,
+ getImageInputCapability: async () => ({ status: 'unknown' as const }),
+ classifyError: vi.fn(),
+ },
'unknown-model',
'openai',
undefined,
@@ -412,8 +424,12 @@ describe('GFS bytes to actual provider request', () => {
undefined,
subject.budget
)
- await expect(fallback.completeWithTools({ messages: shaped, tools: [] })).rejects.toThrow(
- 'Image input is not verified'
+ const fallbackMessages = structuredClone(shaped)
+ await fallback.completeWithTools({ messages: fallbackMessages, tools: [] })
+ expect(completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
+ expect(fallbackMessages.some(m => m.contentParts?.some(p => p.type === 'image'))).toBe(false)
+ expect(JSON.stringify(fallbackMessages)).toContain(
+ 'image_input_not_verified_for_selected_model'
)
expect(sdkCreate).not.toHaveBeenCalled()
await subject.adapter.completeWithTools({ messages: shaped, tools: [] })
@@ -478,6 +494,7 @@ describe('GFS bytes to actual provider request', () => {
Buffer.from('%PDF-1.7'),
Buffer.from([0x50, 0x4b, 0x03, 0x04]),
Buffer.from('<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg"/>'),
+ Buffer.from('<svg/>'),
])('returns a reference instead of interpreting unsupported bytes as text', async bytes => {
const subject = await setup(bytes, { name: 'misleading.txt' })
expect(JSON.parse(subject.output.content).delivery).toBe('reference_only')
@@ -499,14 +516,30 @@ describe('GFS bytes to actual provider request', () => {
it('rechecks the actual destination instead of forwarding images to an unknown fallback', async () => {
const subject = await setup(createCanvas(1, 1).toBuffer('image/png'))
+ const completeSingleTurnWithTools = vi.fn().mockResolvedValue({
+ content: 'degraded',
+ tool_calls: [],
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: 'stop',
+ })
const fallback = new LlmPortAdapter(
- new OpenAIProvider('integration-only', 'unknown-model'),
+ {
+ completeSingleTurn: vi.fn(),
+ completeSingleTurnWithTools,
+ getProviderType: () => 'openai' as const,
+ getImageInputCapability: async () => ({ status: 'unknown' as const }),
+ classifyError: vi.fn(),
+ },
'unknown-model',
'openai'
)
- await expect(
- fallback.completeWithTools({ messages: subject.messages, tools: [] })
- ).rejects.toThrow('Image input is not verified')
+ const fallbackMessages = structuredClone(subject.messages)
+ await fallback.completeWithTools({ messages: fallbackMessages, tools: [] })
+ expect(completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
+ expect(fallbackMessages.some(m => m.contentParts?.some(p => p.type === 'image'))).toBe(false)
+ expect(JSON.stringify(fallbackMessages)).toContain(
+ 'image_input_not_verified_for_selected_model'
+ )
expect(sdkCreate).not.toHaveBeenCalled()
subject.budget.close()
})
diff --git a/mcp-host/src/visualInput/requestPolicy.test.ts b/mcp-host/src/visualInput/requestPolicy.test.ts
index c472d8ca5..c5eaa3656 100644
--- a/mcp-host/src/visualInput/requestPolicy.test.ts
+++ b/mcp-host/src/visualInput/requestPolicy.test.ts
@@ -1,7 +1,7 @@
import { describe, expect, it } from 'vitest'
import type { ChatMessage } from '../core/types'
import { VISUAL_INPUT_LIMITS } from './policy'
-import { assertVisualRequestFits } from './requestPolicy'
+import { assertVisualRequestFits, degradeUnverifiedImageInput } from './requestPolicy'
function messages(count: number): ChatMessage[] {
return [
@@ -44,6 +44,21 @@ describe('last-mile visual request limits', () => {
).toThrow('limit_exceeded')
})
+ it('replaces every image part when the destination cannot take them', () => {
+ const request = messages(2)
+ expect(degradeUnverifiedImageInput(request)).toBe(true)
+ expect(request[0].contentParts?.some(part => part.type === 'image')).toBe(false)
+ expect(
+ JSON.parse(
+ request[0].contentParts![0].type === 'text' ? request[0].contentParts![0].text : '{}'
+ )
+ ).toMatchObject({
+ delivery: 'reference_only',
+ reason: 'image_input_not_verified_for_selected_model',
+ })
+ expect(request[0].content).toContain('not verified')
+ })
+
it('does not change existing text-only requests', () => {
expect(() =>
assertVisualRequestFits(
diff --git a/mcp-host/src/visualInput/requestPolicy.ts b/mcp-host/src/visualInput/requestPolicy.ts
index 3db6d3c67..7fc87c969 100644
--- a/mcp-host/src/visualInput/requestPolicy.ts
+++ b/mcp-host/src/visualInput/requestPolicy.ts
@@ -1,6 +1,38 @@
-import type { ChatMessage } from '../core/types'
+import type { ChatMessage, MessageContentPart } from '../core/types'
import { VISUAL_INPUT_LIMITS, VisualInputError } from './policy'
+function referenceResource(part: MessageContentPart) {
+ if (part.type !== 'image' || !part.source) return undefined
+ const { kind, drive, resourceId, gfsUri, version, name } = part.source
+ return { kind, drive, resourceId, gfsUri, version, name }
+}
+
+/** Replace image parts with references when the destination cannot take them. */
+export function degradeUnverifiedImageInput(messages: ChatMessage[]): boolean {
+ let changed = false
+ for (const message of messages) {
+ if (!message.contentParts?.some(part => part.type === 'image')) continue
+ changed = true
+ message.contentParts = message.contentParts.map(part => {
+ if (part.type !== 'image') return part
+ const resource = referenceResource(part)
+ return {
+ type: 'text' as const,
+ text: JSON.stringify({
+ delivery: 'reference_only',
+ reason: 'image_input_not_verified_for_selected_model',
+ ...(resource ? { resource } : {}),
+ }),
+ }
+ })
+ if (message.role === 'user') {
+ message.content =
+ 'Image input was not verified for the selected model. Treat the listed resources as references only.'
+ }
+ }
+ return changed
+}
+
export function hasGfsImageInput(messages: readonly ChatMessage[]): boolean {
return messages.some(message =>
message.contentParts?.some(part => part.type === 'image' && part.source?.kind === 'gfs')
diff --git a/mcp-host/src/visualInput/suspension.test.ts b/mcp-host/src/visualInput/suspension.test.ts
index 2c129e9d7..9fad35bf3 100644
--- a/mcp-host/src/visualInput/suspension.test.ts
+++ b/mcp-host/src/visualInput/suspension.test.ts
@@ -80,6 +80,7 @@ describe('current-turn GFS image lifecycle', () => {
expect(images.every(p => p.source === undefined)).toBe(true)
const reference = messages.find(m => m.tool_call_id === 'read-1')!
expect(JSON.parse(reference.content).reason).toBe('image_input_limit_exceeded')
+ expect(gfs.attachments).toBeUndefined()
}
)
it('reinserts a reread after the image leaves the context, without collecting it as a generated download', () => {
@@ -163,7 +164,8 @@ describe('current-turn GFS image lifecycle', () => {
],
completed_results: [result()],
attachments: [image, generated],
- }
+ visualPayload: 'must-not-survive-projection',
+ } as PendingApproval & { visualPayload: string }
const projected = projectGfsApproval(approval)
expect(projected.parameters).toBe(approval.parameters)
expect(projected.request_id).toBe(approval.request_id)
@@ -175,6 +177,7 @@ describe('current-turn GFS image lifecycle', () => {
expect(projected.completed_results![0].tool_call_id).toBe('read-1')
expect(projectGfsApproval(projected)).toEqual(projected)
expect(approval.attachments).toHaveLength(2)
+ expect(projected).not.toHaveProperty('visualPayload')
// A later authorized read may produce another version; it must be new input.
appendToolResults(
projected.context_snapshot,
@@ -192,4 +195,35 @@ describe('current-turn GFS image lifecycle', () => {
expect(delivered).toHaveLength(1)
expect(delivered[0].source?.version).toBe(4)
})
+
+ it('projects unsourced image parts and unknown extra approval fields', () => {
+ const approval = {
+ request_id: 'approval-unsourced',
+ tool_name: 'shell_exec',
+ tool_call_id: 'pending-2',
+ description: 'pending operation',
+ parameters: { command: 'echo ready' },
+ context_snapshot: [
+ {
+ role: 'user',
+ content: 'image',
+ contentParts: [{ type: 'image', mimeType: 'image/png', data: image.dataBase64 }],
+ },
+ ],
+ completed_results: [
+ {
+ ...result('shot-1'),
+ name: 'screenshots',
+ attachments: [{ ...image, visualSource: undefined }],
+ },
+ ],
+ visualPayload: 'secret-pixels',
+ } as PendingApproval & { visualPayload: string }
+ const projected = projectGfsApproval(approval)
+ expect(JSON.stringify(projected)).not.toContain(image.dataBase64)
+ expect(JSON.stringify(projected)).not.toContain('secret-pixels')
+ expect(projected).not.toHaveProperty('visualPayload')
+ expect(projected.completed_results![0].attachments).toBeUndefined()
+ expect(projected.context_snapshot[0].contentParts?.some(p => p.type === 'image')).toBe(false)
+ })
})
diff --git a/mcp-host/src/visualInput/suspension.ts b/mcp-host/src/visualInput/suspension.ts
index dd3c4a81f..68ce01cbd 100644
--- a/mcp-host/src/visualInput/suspension.ts
+++ b/mcp-host/src/visualInput/suspension.ts
@@ -13,16 +13,15 @@ function reference(source: GfsImageSource) {
}
function projectMessage(message: ChatMessage): ChatMessage {
- if (!message.contentParts?.some(p => p.type === 'image' && p.source?.kind === 'gfs'))
- return message
+ if (!message.contentParts?.some(p => p.type === 'image')) return message
const parts: MessageContentPart[] = message.contentParts.map(part =>
- part.type === 'image' && part.source?.kind === 'gfs'
+ part.type === 'image'
? {
type: 'text',
text: JSON.stringify({
delivery: 'reference_only',
reason: 'new_gfs_read_required_after_suspension',
- resource: reference(part.source),
+ ...(part.source?.kind === 'gfs' ? { resource: reference(part.source) } : {}),
}),
}
: part
@@ -35,29 +34,55 @@ function projectMessage(message: ChatMessage): ChatMessage {
}
function projectResult(result: ToolResult): ToolResult {
- const removed = result.attachments?.filter(a => a.visualSource?.kind === 'gfs') ?? []
+ const removed =
+ result.attachments?.filter(a => a.kind === 'image' || a.visualSource?.kind === 'gfs') ?? []
if (!removed.length) return result
- const attachments = result.attachments?.filter(a => a.visualSource?.kind !== 'gfs')
+ const attachments = result.attachments?.filter(
+ a => a.kind !== 'image' && a.visualSource?.kind !== 'gfs'
+ )
const content = JSON.stringify({
delivery: 'reference_only',
reason: 'new_gfs_read_required_after_suspension',
- resources: removed.map(a => reference(a.visualSource!)),
+ resources: removed.flatMap(a =>
+ a.visualSource?.kind === 'gfs' ? [reference(a.visualSource)] : []
+ ),
})
return {
- ...result,
+ tool_call_id: result.tool_call_id,
+ name: result.name,
content,
rawContent: content,
- spillover_ref: undefined,
+ is_error: result.is_error,
+ ...(result.metadata ? { metadata: result.metadata } : {}),
attachments: attachments?.length ? attachments : undefined,
}
}
-/** Preserve approval identity/order while removing only newly introduced GFS payloads. */
+/** Preserve approval identity/order while removing every image payload. */
export function projectGfsApproval(approval: PendingApproval): PendingApproval {
return {
- ...approval,
+ request_id: approval.request_id,
+ tool_name: approval.tool_name,
+ tool_call_id: approval.tool_call_id,
+ description: approval.description,
+ parameters: approval.parameters,
context_snapshot: approval.context_snapshot.map(projectMessage),
- completed_results: approval.completed_results?.map(projectResult),
- attachments: approval.attachments?.filter(a => a.visualSource?.kind !== 'gfs'),
+ ...(approval.completed_results
+ ? { completed_results: approval.completed_results.map(projectResult) }
+ : {}),
+ attachments: approval.attachments?.filter(
+ a => a.kind !== 'image' && a.visualSource?.kind !== 'gfs'
+ ),
+ ...(approval.task_budget ? { task_budget: approval.task_budget } : {}),
+ ...(approval.legacy_budget ? { legacy_budget: approval.legacy_budget } : {}),
+ ...(approval.replaces_request_id ? { replaces_request_id: approval.replaces_request_id } : {}),
+ ...(approval.tool_kind ? { tool_kind: approval.tool_kind } : {}),
+ ...(approval.tool_source_ref !== undefined
+ ? { tool_source_ref: approval.tool_source_ref }
+ : {}),
+ ...(approval.intent_summary ? { intent_summary: approval.intent_summary } : {}),
+ ...(approval.traceContext !== undefined ? { traceContext: approval.traceContext } : {}),
+ ...(approval.reason ? { reason: approval.reason } : {}),
+ ...(approval.mcpServerName ? { mcpServerName: approval.mcpServerName } : {}),
}
}