Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): support documented OpenAI image input contracts
Alfredo Lopezd591b2c3472b
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 238251a96..f05a8e63b 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.339",+ "version": "0.1.340","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.339",+ "version": "0.1.340","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 7ab100ca0..0b6cff67a 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.339",+ "version": "0.1.340","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/internalTools/gfsImageInput.integration.test.ts b/mcp-host/src/internalTools/gfsImageInput.integration.test.tsindex 6bff95a76..4ced2cd7f 100644--- a/mcp-host/src/internalTools/gfsImageInput.integration.test.ts+++ b/mcp-host/src/internalTools/gfsImageInput.integration.test.ts@@ -165,6 +165,43 @@ afterEach(() => {})describe('GFS bytes to actual provider request', () => {+ it.each(['image/png', 'image/jpeg'] as const)(+ 'delivers %s directly to a documented OpenAI model',+ async mime => {+ const model = 'gpt-4.1'+ // Only the SDK HTTP boundary is doubled; no credential or catalog service.+ const client = {+ baseURL: 'https://api.openai.com/v1',+ chat: { completions: { create: sdkCreate } },+ }+ const canvas = createCanvas(8, 8)+ canvas.getContext('2d').fillRect(1, 1, 4, 4)+ const bytes =+ mime === 'image/png' ? canvas.toBuffer('image/png') : canvas.toBuffer('image/jpeg')+ const subject = await setup(bytes, {+ llm: {+ provider: new OpenAIProvider(client as never, model),+ model,+ name: 'openai',+ },+ })+ expect(JSON.parse(subject.output.content).delivery).toBe('image_input')+ await subject.adapter.completeWithTools({ messages: subject.messages, tools: [] })+ expect(subject.metadataFetch).not.toHaveBeenCalled()+ const request = sdkCreate.mock.calls[0][0]+ const images = request.messages+ .flatMap((message: any) => (Array.isArray(message.content) ? message.content : []))+ .filter((part: any) => part.type === 'image_url')+ expect(images).toEqual([+ {+ type: 'image_url',+ image_url: { url: `data:${mime};base64,${bytes.toString('base64')}` },+ },+ ])+ expect(request.model).toBe(model)+ subject.budget.close()+ }+ )it.each([['image/png', false],['image/jpeg', false],diff --git a/mcp-host/src/llm/__tests__/openai.singleTurn.test.ts b/mcp-host/src/llm/__tests__/openai.singleTurn.test.tsindex 7827e34ed..cf3cce165 100644--- a/mcp-host/src/llm/__tests__/openai.singleTurn.test.ts+++ b/mcp-host/src/llm/__tests__/openai.singleTurn.test.ts@@ -2,6 +2,30 @@ import { describe, expect, it, vi } from 'vitest'import { type ChatMessage, FinishReason, type ToolDefinition } from '../../core/types'import { OpenAIProvider } from '../openai'+describe('OpenAI image capability binding', () => {+ it('binds the documented capability to the actual SDK endpoint', async () => {+ const client = { ...createMockOpenAIClient(), baseURL: 'https://api.openai.com/v1' }+ const provider = new OpenAIProvider(client as never, 'gpt-4.1')+ await expect(provider.getImageInputCapability()).resolves.toMatchObject({+ status: 'supported',+ provider: 'openai',+ model: 'gpt-4.1',+ })+ client.baseURL = 'https://gateway.example.invalid/v1'+ await expect(provider.getImageInputCapability()).resolves.toEqual({ status: 'unknown' })+ expect(client.chat.completions.create).not.toHaveBeenCalled()+ })++ it('honors cancellation before resolving a capability', async () => {+ const client = { ...createMockOpenAIClient(), baseURL: 'https://api.openai.com/v1' }+ const provider = new OpenAIProvider(client as never, 'gpt-4.1')+ await expect(provider.getImageInputCapability(AbortSignal.abort())).rejects.toMatchObject({+ code: 'cancelled',+ })+ expect(client.chat.completions.create).not.toHaveBeenCalled()+ })+})+function createMockOpenAIClient() {return {chat: {diff --git a/mcp-host/src/llm/__tests__/openaiCompatible.classifyError.test.ts b/mcp-host/src/llm/__tests__/openaiCompatible.classifyError.test.tsindex c2743bd72..f5d2e63f4 100644--- a/mcp-host/src/llm/__tests__/openaiCompatible.classifyError.test.ts+++ b/mcp-host/src/llm/__tests__/openaiCompatible.classifyError.test.ts@@ -16,6 +16,14 @@ const bailian = new OpenAICompatibleProvider(// lands at `err.error.code`, the level the override reads. Deriving guards// against the same nesting mismatch that broke the Claude arm.describe('OpenAICompatibleProvider.classifyError', () => {+ it('does not inherit OpenAI model evidence for a different configured provider', async () => {+ // Construction-only fixture; no authenticated request is made.+ const compatible = new OpenAICompatibleProvider(+ { id: 'zai', baseURL: 'https://api.openai.com/v1', defaultModel: 'gpt-4.1' },+ 'unit-only'+ )+ await expect(compatible.getImageInputCapability()).resolves.toEqual({ status: 'unknown' })+ })it('maps z.ai model-not-available codes (1211/1220) to ModelNotAvailable', () => {for (const code of ['1211', '1220']) {const c = zai.classifyError(openaiApiError(404, { code, message: 'model not available' }))diff --git a/mcp-host/src/llm/openai.ts b/mcp-host/src/llm/openai.tsindex 365907059..aa588d54f 100644--- a/mcp-host/src/llm/openai.ts+++ b/mcp-host/src/llm/openai.ts@@ -10,6 +10,8 @@ import {ToolDefinition,} from '../core/types'import { logger } from '../logger'+import { getDocumentedOpenAIImageCapability } from '../visualInput/documentedCapabilities'+import { type ImageInputCapability, VisualInputError } from '../visualInput/policy'import { assertVisualRequestFits } from '../visualInput/requestPolicy'import { classifyByHttpStatus, classifyUnknown } from './errorClassification'import type { LlmProvider } from './registryCore'@@ -29,6 +31,13 @@ export class OpenAIProvider implements SingleTurnProvider {logger.info({ model: defaultModel }, 'OpenAI transport initialized')}+ async getImageInputCapability(signal?: AbortSignal): Promise<ImageInputCapability> {+ if (signal?.aborted) throw new VisualInputError('cancelled')+ // OpenAI's Models API does not report modalities. Use the documented+ // contract for this exact model only when this SDK targets the official API.+ return getDocumentedOpenAIImageCapability(this.defaultModel, this.client.baseURL)+ }+/*** Get the provider type.*/diff --git a/mcp-host/src/llm/openaiCompatible.ts b/mcp-host/src/llm/openaiCompatible.tsindex 758c5e7e9..53e570315 100644--- a/mcp-host/src/llm/openaiCompatible.ts+++ b/mcp-host/src/llm/openaiCompatible.ts@@ -126,7 +126,7 @@ export class OpenAICompatibleProvider extends OpenAIProvider {return this.cfg.id as LlmProvider}- getImageInputCapability(signal?: AbortSignal): Promise<ImageInputCapability> {+ override getImageInputCapability(signal?: AbortSignal): Promise<ImageInputCapability> {return this.imageCapabilityResolver?.(signal) ?? Promise.resolve({ status: 'unknown' })}diff --git a/mcp-host/src/visualInput/documentedCapabilities.test.ts b/mcp-host/src/visualInput/documentedCapabilities.test.tsnew file mode 100644index 000000000..6e215a952--- /dev/null+++ b/mcp-host/src/visualInput/documentedCapabilities.test.ts@@ -0,0 +1,187 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import {+ DOCUMENTED_OPENAI_IMAGE_MODELS,+ OFFICIAL_OPENAI_BASE_URL,+ OPENAI_DOCUMENTED_EVIDENCE,+ OPENAI_PROVIDER,+ getDocumentedOpenAIImageCapability,+} from './documentedCapabilities'++const OFFICIAL = OFFICIAL_OPENAI_BASE_URL+/** The exact default the OpenAI SDK resolves for an absent base URL. */+const SDK_DEFAULT_BASE_URL = 'https://api.openai.com/v1'+const DOCUMENTED_MODEL = 'gpt-4.1'++afterEach(() => {+ vi.restoreAllMocks()+ vi.unstubAllGlobals()+})++function expectSupported(model: string, baseURL: string | undefined): void {+ expect(getDocumentedOpenAIImageCapability(model, baseURL)).toEqual({+ status: 'supported',+ provider: 'openai',+ model,+ evidence: 'openai-documented-model-contract',+ })+}++function expectUnknown(model: string, baseURL: string | undefined = OFFICIAL): void {+ expect(getDocumentedOpenAIImageCapability(model, baseURL)).toEqual({ status: 'unknown' })+}++describe('getDocumentedOpenAIImageCapability', () => {+ it('reports the documented contract for every table entry, alias and snapshot alike', () => {+ expect(OPENAI_PROVIDER).toBe('openai')+ expect(OPENAI_DOCUMENTED_EVIDENCE).toBe('openai-documented-model-contract')+ expect(OFFICIAL_OPENAI_BASE_URL).toBe('https://api.openai.com/v1')+ expect(DOCUMENTED_OPENAI_IMAGE_MODELS.length).toBeGreaterThan(0)++ for (const entry of DOCUMENTED_OPENAI_IMAGE_MODELS) {+ expectSupported(entry.id, OFFICIAL)+ // The SDK default and the explicit official base must agree.+ expectSupported(entry.id, SDK_DEFAULT_BASE_URL)+ expect(getDocumentedOpenAIImageCapability(entry.id, undefined)).toEqual({ status: 'unknown' })+ for (const snapshot of entry.snapshots) expectSupported(snapshot, OFFICIAL)+ }+ })++ it('answers for the model the OpenAI provider actually defaults to', () => {+ expectSupported('gpt-5.4-mini', SDK_DEFAULT_BASE_URL)+ expectSupported('gpt-5.4-mini-2026-03-17', SDK_DEFAULT_BASE_URL)+ })++ it('keeps one source of truth per documented id, with no collisions', () => {+ const seen = new Map<string, string>()+ for (const entry of DOCUMENTED_OPENAI_IMAGE_MODELS) {+ expect(entry.source).toBe(`https://developers.openai.com/api/docs/models/${entry.id}`)+ for (const id of [entry.id, ...entry.snapshots]) {+ expect(seen.has(id)).toBe(false)+ seen.set(id, entry.source)+ }+ for (const snapshot of entry.snapshots) {+ // A snapshot of another model must never be reachable through this entry.+ expect(snapshot.startsWith(`${entry.id}-`)).toBe(true)+ }+ }+ // Every documented id the getter accepts is covered by the table above.+ expect([...seen.keys()]).toContain(DOCUMENTED_MODEL)+ })++ it('binds the answer to the official endpoint, with one trailing slash at most', () => {+ expectSupported(DOCUMENTED_MODEL, 'https://api.openai.com/v1')+ expectSupported(DOCUMENTED_MODEL, 'https://api.openai.com/v1/')+ // Host names are case-insensitive; the explicit default port is the same origin.+ expectSupported(DOCUMENTED_MODEL, 'https://API.OPENAI.COM/v1')+ expectSupported(DOCUMENTED_MODEL, 'https://api.openai.com:443/v1')+ })++ it('returns unknown for every other origin, path, scheme, or credentials', () => {+ const rejected = [+ // Different scheme, host, or port: a different deployment.+ 'http://api.openai.com/v1',+ 'https://api.openai.com.evil.example/v1',+ 'https://api.openai.com.evil.example./v1',+ 'https://evil.example/?next=https://api.openai.com/v1',+ 'https://evil.example/api.openai.com/v1',+ 'https://api.openai.com:8443/v1',+ 'https://api.openai.com./v1',+ // Credentials, query, or fragment change the request target.+ 'https://user:[email protected]/v1',+ 'https://[email protected]/v1',+ 'https://api.openai.com/v1?x=1',+ 'https://api.openai.com/v1#fragment',+ 'https://api.openai.com/v1#@evil.example',+ // Paths that are not the documented base.+ 'https://api.openai.com',+ 'https://api.openai.com/',+ 'https://api.openai.com/v1//',+ 'https://api.openai.com/v1/chat/completions',+ 'https://api.openai.com/v1/models',+ 'https://api.openai.com/v2',+ 'https://api.openai.com/V1',+ // Other documented OpenAI base URLs and gateways.+ 'https://openrouter.ai/api/v1',+ 'https://example.openai.azure.com/openai/deployments/gpt-4.1',+ 'http://127.0.0.1:11434/v1',+ 'https://api.anthropic.com/v1',+ // Not a usable absolute base URL.+ '',+ '/v1',+ 'api.openai.com/v1',+ 'not a url',+ ]++ for (const baseURL of rejected) expectUnknown(DOCUMENTED_MODEL, baseURL)+ })++ it('never infers support from a prefix, family, or neighbouring id', () => {+ const undocumented = [+ // Same families, ids that are not in the table.+ 'gpt-4',+ 'gpt-4-turbo',+ 'gpt-4.1-preview',+ 'gpt-4.1-2025-04-14x',+ 'gpt-4.1-20250414',+ 'gpt-4o-2024-05-14',+ 'gpt-4o-audio-preview',+ 'gpt-4o-mini-realtime-preview',+ 'gpt-4o-transcribe',+ 'gpt-5-pro',+ 'gpt-5.2-pro',+ 'gpt-5.5-pro',+ 'gpt-5.5',+ 'gpt-5.4-pro',+ 'gpt-5.4-mini-2026-03-17x',+ 'gpt-5.1-codex',+ 'gpt-5.1-codex-max',+ 'gpt-5.3-codex-spark',+ 'gpt-6',+ 'o3',+ 'o4-mini',+ 'text-embedding-3-large',+ 'gpt-image-1',+ 'dall-e-3',+ 'whisper-1',+ 'tts-1',+ // Adjacent or injected spellings of a documented id.+ 'gpt-4.1 ',+ ' gpt-4.1',+ 'gpt-4.1\n',+ 'gpt-4.1\t',+ 'GPT-4.1',+ 'xgpt-4.1',+ 'gpt-4.1/../gpt-4.1',+ 'gpt-4.1%00',+ 'gpt-4.1-2025-04-14.',+ 'gpt-4.1-evil',+ '',+ ' ',+ ]++ for (const model of undocumented) expectUnknown(model)+ })++ it('does not trim, fold, or otherwise normalize the requested id', () => {+ // The documented id is answered verbatim; the same letters with any added+ // character are a different id and stay unknown.+ expect(getDocumentedOpenAIImageCapability(` ${DOCUMENTED_MODEL}`, OFFICIAL)).toEqual({+ status: 'unknown',+ })+ expect(getDocumentedOpenAIImageCapability(DOCUMENTED_MODEL.toUpperCase(), OFFICIAL)).toEqual({+ status: 'unknown',+ })+ expectSupported(DOCUMENTED_MODEL, OFFICIAL)+ })++ it('answers without any network request or credential', () => {+ const fetchSpy = vi.fn()+ vi.stubGlobal('fetch', fetchSpy)++ expectSupported(DOCUMENTED_MODEL, OFFICIAL)+ expectUnknown('gpt-4.1-nope')+ expectUnknown(DOCUMENTED_MODEL, 'https://proxy.example/v1')++ expect(fetchSpy).not.toHaveBeenCalled()+ })+})diff --git a/mcp-host/src/visualInput/documentedCapabilities.ts b/mcp-host/src/visualInput/documentedCapabilities.tsnew file mode 100644index 000000000..1bd8ec675--- /dev/null+++ b/mcp-host/src/visualInput/documentedCapabilities.ts@@ -0,0 +1,189 @@+/**+ * Documented OpenAI image-input capability, resolved offline.+ *+ * OpenAI's Models API reports an id and an owner but not modalities+ * (https://developers.openai.com/api/reference/cli/resources/models/methods/retrieve),+ * so there is nothing to read back at runtime. Instead this module binds one+ * factual table of published model contracts to the one endpoint those contracts+ * were published for. A model is image-capable only when both hold:+ *+ * 1. the transport targets the official `https://api.openai.com/v1` endpoint, and+ * 2. the exact requested id appears in {@link DOCUMENTED_OPENAI_IMAGE_MODELS}.+ *+ * Everything else is `unknown`. There is no prefix, family, or size heuristic:+ * an id is listed only when a verified page documents `Image: Input only` *and*+ * the id is in scope for this provider's Chat Completions path. Codex variants,+ * Pro variants, audio/realtime models, and every unlisted id therefore stay+ * `unknown`; some of them (for example `gpt-5-pro` and `gpt-5.1-codex`) document+ * image input on their own pages, which on its own is not enough to list them+ * here. The table is a contract with sources, not a catalog: each entry carries+ * the official page it was verified against and the snapshot ids that page+ * lists.+ *+ * This module performs no I/O, reads no key, and never infers capability from a+ * model name. Absence of an entry means "not proven", never "not capable".+ */+import type { ImageInputCapability } from './policy'++/** The provider identity reported for a supported model. */+export const OPENAI_PROVIDER = 'openai'+/**+ * Fixed, bounded provenance string. The model id is echoed back exactly as+ * requested, so a caller can correlate the answer with its own request; the+ * evidence never contains upstream text.+ */+export const OPENAI_DOCUMENTED_EVIDENCE = 'openai-documented-model-contract'+/** The only endpoint whose model contracts this table describes. */+export const OFFICIAL_OPENAI_BASE_URL = 'https://api.openai.com/v1'++/**+ * One verified model contract: the documented model id, the snapshot ids the+ * same page lists for it, and the official page they were read from.+ */+export interface DocumentedOpenAIImageModel {+ /** Documented model id, accepted verbatim as `model` by the Chat Completions API. */+ readonly id: string+ /** Snapshot ids listed for {@link id} on the same page; empty for pointer aliases. */+ readonly snapshots: readonly string[]+ /** Official page this contract was verified against. */+ readonly source: string+}++/**+ * Verified on 2026-09-17. Every entry's page documents image input and the+ * snapshot ids listed here. This is provider-local contract data, not a+ * replacement for discovery or the broader capability catalog in #654.+ */+export const DOCUMENTED_OPENAI_IMAGE_MODELS: readonly DocumentedOpenAIImageModel[] = Object.freeze([+ Object.freeze({+ id: 'gpt-4.1',+ snapshots: Object.freeze(['gpt-4.1-2025-04-14']),+ source: 'https://developers.openai.com/api/docs/models/gpt-4.1',+ }),+ Object.freeze({+ id: 'gpt-4.1-mini',+ snapshots: Object.freeze(['gpt-4.1-mini-2025-04-14']),+ source: 'https://developers.openai.com/api/docs/models/gpt-4.1-mini',+ }),+ Object.freeze({+ id: 'gpt-4.1-nano',+ snapshots: Object.freeze(['gpt-4.1-nano-2025-04-14']),+ source: 'https://developers.openai.com/api/docs/models/gpt-4.1-nano',+ }),+ Object.freeze({+ id: 'gpt-4o',+ snapshots: Object.freeze(['gpt-4o-2024-05-13', 'gpt-4o-2024-08-06', 'gpt-4o-2024-11-20']),+ source: 'https://developers.openai.com/api/docs/models/gpt-4o',+ }),+ Object.freeze({+ id: 'gpt-4o-mini',+ snapshots: Object.freeze(['gpt-4o-mini-2024-07-18']),+ source: 'https://developers.openai.com/api/docs/models/gpt-4o-mini',+ }),+ Object.freeze({+ id: 'gpt-5.4-mini',+ snapshots: Object.freeze(['gpt-5.4-mini-2026-03-17']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.4-mini',+ }),+ Object.freeze({+ id: 'gpt-5.4',+ snapshots: Object.freeze(['gpt-5.4-2026-03-05']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.4',+ }),+ Object.freeze({+ id: 'gpt-5.4-nano',+ snapshots: Object.freeze(['gpt-5.4-nano-2026-03-17']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.4-nano',+ }),+ Object.freeze({+ id: 'gpt-5',+ snapshots: Object.freeze(['gpt-5-2025-08-07']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5',+ }),+ Object.freeze({+ id: 'gpt-5-mini',+ snapshots: Object.freeze(['gpt-5-mini-2025-08-07']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5-mini',+ }),+ Object.freeze({+ id: 'gpt-5-nano',+ snapshots: Object.freeze(['gpt-5-nano-2025-08-07']),+ source: 'https://developers.openai.com/api/docs/models/gpt-5-nano',+ }),+ Object.freeze({+ id: 'gpt-5-chat-latest',+ snapshots: Object.freeze([]),+ source: 'https://developers.openai.com/api/docs/models/gpt-5-chat-latest',+ }),+ Object.freeze({+ id: 'gpt-5.1-chat-latest',+ snapshots: Object.freeze([]),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.1-chat-latest',+ }),+ Object.freeze({+ id: 'gpt-5.2-chat-latest',+ snapshots: Object.freeze([]),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.2-chat-latest',+ }),+ Object.freeze({+ id: 'gpt-5.3-chat-latest',+ snapshots: Object.freeze([]),+ source: 'https://developers.openai.com/api/docs/models/gpt-5.3-chat-latest',+ }),+])++const UNKNOWN = Object.freeze({ status: 'unknown' as const })++/** Aliases and snapshots share one lookup: both are documented model ids. */+const DOCUMENTED_MODEL_IDS: ReadonlySet<string> = new Set(+ DOCUMENTED_OPENAI_IMAGE_MODELS.flatMap(entry => [entry.id, ...entry.snapshots])+)++/**+ * Reads the documented image-input contract for one exact model id. Returns+ * `supported` with the requested id only when the transport targets the official+ * OpenAI endpoint and this module has a verified contract for that id;+ * otherwise `unknown`, so an unproven model is never handed an image.+ */+export function getDocumentedOpenAIImageCapability(+ model: string,+ baseURL: string | undefined+): ImageInputCapability {+ if (!targetsOfficialOpenAiEndpoint(baseURL)) return UNKNOWN+ // Exact membership only: no trimming, no case folding, no prefix family.+ if (!DOCUMENTED_MODEL_IDS.has(model)) return UNKNOWN++ return Object.freeze({+ status: 'supported' as const,+ provider: OPENAI_PROVIDER,+ model,+ evidence: OPENAI_DOCUMENTED_EVIDENCE,+ })+}++/**+ * True only for the resolved official `https://api.openai.com/v1` base. This+ * receives the SDK instance's effective URL, not its optional constructor input.+ * An absent endpoint is therefore unverified. Any other origin, path,+ * scheme, port, or a URL carrying userinfo, query, or fragment is a different+ * deployment whose model contracts this table does not describe.+ */+function targetsOfficialOpenAiEndpoint(baseURL: string | undefined): boolean {+ if (baseURL === undefined) return false++ let url: URL+ try {+ url = new URL(baseURL)+ } catch {+ // A relative, empty, or malformed base URL cannot be proven official.+ return false+ }+ if (url.protocol !== 'https:') return false+ if (url.username !== '' || url.password !== '') return false+ if (url.search !== '' || url.hash !== '') return false+ if (url.hostname !== 'api.openai.com') return false+ if (url.port !== '' && url.port !== '443') return false+ // One optional trailing slash is equivalent; `//` and every other path are not.+ const path = url.pathname.endsWith('/') ? url.pathname.slice(0, -1) : url.pathname+ return path === '/v1'+}