Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(codex): preserve and authorize image input (#650)

Alfredo Lopezc9a63e3b44f1
diff --git a/codex-llm-proxy/src/codexTransport.ts b/codex-llm-proxy/src/codexTransport.ts
index af7b60c5f..75d578628 100644
--- a/codex-llm-proxy/src/codexTransport.ts
+++ b/codex-llm-proxy/src/codexTransport.ts
@@ -1,8 +1,8 @@
import {
- type CodexCompletionRequestV1,
+ type CodexCompletionRequest,
LIMITS,
- hashCodexCompletionRequestV1,
- parseCodexCompletionRequestV1,
+ hashCodexCompletionRequest,
+ parseCodexCompletionRequest,
} from '@clerum/llm-provider-attempt-contract'
import { chatgptUpstreamHeaders } from './chatgptUpstreamHeaders.js'
import type { FinalizeAttemptSuccess, RedeemAttemptSuccess } from './controlApiClient.js'
@@ -42,6 +42,8 @@ export class CodexTransportError extends Error {
}
export type StreamCodexCompletionInput = {
+ /** Explicit rollout gate: local format support is not upstream certification. */
+ imageInputEnabled?: boolean
executionTicket: string
requestHash: string
request: unknown
@@ -79,18 +81,34 @@ export type StreamCodexCompletionResult = {
export async function streamCodexCompletion(
input: StreamCodexCompletionInput
): Promise<StreamCodexCompletionResult> {
- const parsed = parseCodexCompletionRequestV1(input.request)
+ const parsed = parseCodexCompletionRequest(input.request)
if (!parsed.ok) {
throw new CodexTransportError('invalid_request', parsed.message)
}
const request = parsed.value
- const digest = hashCodexCompletionRequestV1(request)
+ if (request.schemaVersion === 'codex-completion-request.v2' && input.deadlineMs !== undefined) {
+ throw new CodexTransportError(
+ 'invalid_request',
+ 'Visual request deadlines must be inside the authorized request'
+ )
+ }
+ const digest = hashCodexCompletionRequest(request)
if (digest !== input.requestHash || input.ticket.requestHash !== input.requestHash) {
throw new CodexTransportError('request_hash_mismatch', 'request hash does not match the ticket')
}
if (request.model !== input.ticket.model) {
throw new CodexTransportError('model_not_allowed', 'request model does not match the ticket')
}
+ if (
+ request.schemaVersion === 'codex-completion-request.v2' &&
+ request.messages.some(message => message.contentParts?.some(part => part.type === 'image')) &&
+ input.imageInputEnabled !== true
+ ) {
+ throw new CodexTransportError(
+ 'image_input_unsupported',
+ 'Image input is not enabled for this Codex model'
+ )
+ }
const redeemed = await input.redeem({
executionTicket: input.executionTicket,
requestHash: input.requestHash,
@@ -182,7 +200,7 @@ async function finalizeQuietly(
}
async function readUpstreamStream(input: {
- request: CodexCompletionRequestV1
+ request: CodexCompletionRequest
accessToken: string
chatgptAccountId?: string
deadlineMs: number
@@ -248,7 +266,7 @@ async function readUpstreamStream(input: {
}
function toUpstreamPayload(
- request: CodexCompletionRequestV1,
+ request: CodexCompletionRequest,
names: ToolNameMap
): Record<string, unknown> {
const instructions = request.messages
@@ -280,7 +298,20 @@ function toUpstreamPayload(
}
continue
}
- input.push({ role: message.role, content: message.content })
+ if ('contentParts' in message && message.contentParts) {
+ // Responses content items, not Chat Completions image_url objects.
+ // Provenance remains bound by the local hash and never becomes model text.
+ input.push({
+ role: message.role,
+ content: message.contentParts.map(part =>
+ part.type === 'text'
+ ? { type: 'input_text', text: part.text }
+ : { type: 'input_image', image_url: `data:${part.mimeType};base64,${part.data}` }
+ ),
+ })
+ } else {
+ input.push({ role: message.role, content: message.content })
+ }
}
const payload: Record<string, unknown> = {
model: request.model,
diff --git a/codex-llm-proxy/src/config.ts b/codex-llm-proxy/src/config.ts
index dee88f64e..ebe1ade2b 100644
--- a/codex-llm-proxy/src/config.ts
+++ b/codex-llm-proxy/src/config.ts
@@ -1,3 +1,5 @@
+import { LIMITS } from '@clerum/llm-provider-attempt-contract'
+
export type CodexLlmProxyConfig = {
runtimePort: number
adminPort: number
@@ -8,6 +10,7 @@ export type CodexLlmProxyConfig = {
jwtIssuer: string
jwtPublicKey: string
executionEnabled: boolean
+ imageInputModels: string[]
controlApiBaseUrl: string
controlApiServiceName: string
controlApiServiceToken: string
@@ -33,15 +36,35 @@ function requiredPem(name: string, raw: string | undefined): string {
}
export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyConfig {
+ const imageInputModels = (env.CODEX_IMAGE_INPUT_MODELS ?? '')
+ .split(',')
+ .map(model => model.trim())
+ .filter(Boolean)
+ const maxBodyBytes = requiredPositiveInt(
+ 'CODEX_LLM_PROXY_MAX_BODY_BYTES',
+ env.CODEX_LLM_PROXY_MAX_BODY_BYTES,
+ LIMITS.maxRequestBodyBytes
+ )
+ if (imageInputModels.length > 0 && maxBodyBytes < LIMITS.maxRequestBodyBytes) {
+ throw new Error('Visual Codex requests require the full shared envelope byte budget')
+ }
return {
- runtimePort: requiredPositiveInt('CODEX_LLM_PROXY_RUNTIME_PORT', env.CODEX_LLM_PROXY_RUNTIME_PORT, 8080),
- adminPort: requiredPositiveInt('CODEX_LLM_PROXY_ADMIN_PORT', env.CODEX_LLM_PROXY_ADMIN_PORT, 8081),
- probePort: requiredPositiveInt('CODEX_LLM_PROXY_PROBE_PORT', env.CODEX_LLM_PROXY_PROBE_PORT, 9090),
- maxBodyBytes: requiredPositiveInt(
- 'CODEX_LLM_PROXY_MAX_BODY_BYTES',
- env.CODEX_LLM_PROXY_MAX_BODY_BYTES,
- 1_048_576
+ runtimePort: requiredPositiveInt(
+ 'CODEX_LLM_PROXY_RUNTIME_PORT',
+ env.CODEX_LLM_PROXY_RUNTIME_PORT,
+ 8080
+ ),
+ adminPort: requiredPositiveInt(
+ 'CODEX_LLM_PROXY_ADMIN_PORT',
+ env.CODEX_LLM_PROXY_ADMIN_PORT,
+ 8081
+ ),
+ probePort: requiredPositiveInt(
+ 'CODEX_LLM_PROXY_PROBE_PORT',
+ env.CODEX_LLM_PROXY_PROBE_PORT,
+ 9090
),
+ maxBodyBytes,
maxStreamDurationMs: requiredPositiveInt(
'CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS',
env.CODEX_LLM_PROXY_MAX_STREAM_DURATION_MS,
@@ -55,6 +78,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): CodexLlmProxyC
jwtIssuer: env.CODEX_LLM_PROXY_JWT_ISSUER?.trim() || 'control-api',
jwtPublicKey: requiredPem('CODEX_LLM_PROXY_JWT_PUBLIC_KEY', env.CODEX_LLM_PROXY_JWT_PUBLIC_KEY),
executionEnabled: env.CODEX_LLM_PROXY_EXECUTION_ENABLED === 'true',
+ imageInputModels,
controlApiBaseUrl: env.CODEX_LLM_PROXY_CONTROL_API_URL?.trim() || '',
controlApiServiceName: env.CODEX_LLM_PROXY_CONTROL_API_SERVICE?.trim() || 'codex-llm-proxy',
controlApiServiceToken: env.CODEX_LLM_PROXY_CONTROL_API_TOKEN?.trim() || '',
diff --git a/codex-llm-proxy/src/server.ts b/codex-llm-proxy/src/server.ts
index 219a780d6..9f03c1b25 100644
--- a/codex-llm-proxy/src/server.ts
+++ b/codex-llm-proxy/src/server.ts
@@ -1,6 +1,6 @@
import express, { type Express, type Request, type Response } from 'express'
import { rateLimit } from 'express-rate-limit'
-import { createServer, type Server } from 'node:http'
+import { type Server, createServer } from 'node:http'
import { Registry, collectDefaultMetrics } from 'prom-client'
import { z } from 'zod'
import { verifyAdminPermit } from './auth/adminPermitVerifier.js'
@@ -17,9 +17,9 @@ import { ControlApiClient, ControlApiClientError } from './controlApiClient.js'
import { logger } from './logger.js'
import { createProxyMetrics } from './metrics.js'
import {
- defaultAddressLookup,
OriginDeniedError,
type OriginPolicyOptions,
+ defaultAddressLookup,
} from './originPolicy.js'
import { RequestLimitError, streamGate } from './requestLimits.js'
@@ -84,7 +84,10 @@ export type ProxyServers = {
close: () => Promise<void>
}
-export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeDeps = {}): ProxyServers {
+export function createProxyApps(
+ config: CodexLlmProxyConfig,
+ deps: ProxyRuntimeDeps = {}
+): ProxyServers {
const metricsRegistry = new Registry()
collectDefaultMetrics({ register: metricsRegistry })
const metrics = createProxyMetrics(metricsRegistry)
@@ -168,6 +171,7 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
res.setHeader('cache-control', 'no-cache')
const started = Date.now()
const result = await streamCodexCompletion({
+ imageInputEnabled: config.imageInputModels.includes(ticket.model),
executionTicket: parsed.data.executionTicket,
requestHash: parsed.data.requestHash,
request: parsed.data.request,
@@ -223,7 +227,13 @@ export function createProxyApps(config: CodexLlmProxyConfig, deps: ProxyRuntimeD
reject(res, 403, 'insufficient_scope')
return
}
- if (!verifyAdminPermit(bearer(req), config, kind === 'models' ? 'catalog_list' : 'connection_test')) {
+ if (
+ !verifyAdminPermit(
+ bearer(req),
+ config,
+ kind === 'models' ? 'catalog_list' : 'connection_test'
+ )
+ ) {
reject(res, 401, 'Unauthorized')
return
}
diff --git a/codex-llm-proxy/test/codexTransport.conformance.test.ts b/codex-llm-proxy/test/codexTransport.conformance.test.ts
index 2d9f1d9ab..c2a202407 100644
--- a/codex-llm-proxy/test/codexTransport.conformance.test.ts
+++ b/codex-llm-proxy/test/codexTransport.conformance.test.ts
@@ -1,6 +1,9 @@
import { describe, expect, it, vi } from 'vitest'
+import { readFileSync } from 'node:fs'
import {
+ hashCodexCompletionRequest,
hashCodexCompletionRequestV1,
+ parseCodexCompletionRequest,
parseCodexCompletionRequestV1,
} from '@clerum/llm-provider-attempt-contract'
import { CodexTransportError, streamCodexCompletion } from '../src/codexTransport.js'
@@ -59,6 +62,71 @@ function sseResponse(
}
describe('streamCodexCompletion', () => {
+ it.each(['png', 'jpeg'])(
+ 'projects authorized %s parts without leaking provenance upstream',
+ async format => {
+ const fixtures = JSON.parse(
+ readFileSync(
+ new URL(
+ '../../packages/llm-provider-attempt-contract/fixtures/visual-requests.json',
+ import.meta.url
+ ),
+ 'utf8'
+ )
+ )
+ const parsed = parseCodexCompletionRequest({ ...fixtures[format], model: REQUEST.model })
+ expect(parsed.ok).toBe(true)
+ if (!parsed.ok) throw new Error(parsed.message)
+ const request = parsed.value
+ const requestHash = hashCodexCompletionRequest(request)
+ const fetchFn = vi.fn(async () => sseResponse(['data: {"type":"response.completed"}\n\n']))
+ const redeem = vi.fn(async () => redeemSuccess())
+ const finalize = vi.fn(async () => ({
+ providerAttemptId: 'visual-attempt',
+ outcome: 'success' as const,
+ duplicate: false,
+ }))
+ const input = {
+ executionTicket: 'visual-ticket',
+ requestHash,
+ request,
+ ticket: {
+ jti: 'visual-ticket',
+ hostRef: 'research-host',
+ model: request.model,
+ requestHash,
+ providerAttemptId: 'visual-attempt',
+ },
+ redeem,
+ finalize,
+ fetchFn,
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ }
+ await expect(streamCodexCompletion(input)).rejects.toMatchObject({
+ code: 'image_input_unsupported',
+ })
+ expect(redeem).not.toHaveBeenCalled()
+ expect(fetchFn).not.toHaveBeenCalled()
+ await expect(
+ streamCodexCompletion({ ...input, imageInputEnabled: true, deadlineMs: 1000 })
+ ).rejects.toMatchObject({ code: 'invalid_request' })
+ expect(redeem).not.toHaveBeenCalled()
+ await streamCodexCompletion({ ...input, imageInputEnabled: true })
+ const body = JSON.parse(String(fetchFn.mock.calls[0]?.[1]?.body))
+ const sourceParts = fixtures[format].messages[0].contentParts
+ expect(body.input[0].content).toEqual(
+ sourceParts.map(
+ (part: { type: string; text?: string; mimeType?: string; data?: string }) =>
+ part.type === 'text'
+ ? { type: 'input_text', text: part.text }
+ : { type: 'input_image', image_url: `data:${part.mimeType};base64,${part.data}` }
+ )
+ )
+ expect(JSON.stringify(body)).not.toContain('attachmentId')
+ expect(finalize).toHaveBeenCalledOnce()
+ }
+ )
+
it('preserves real MCP optional schemas with explicit non-strict upstream tools', async () => {
const request = {
...REQUEST,
diff --git a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
index 413ab4d95..902d8e253 100644
--- a/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
+++ b/codex-llm-proxy/test/runtimePath.hermetic.e2e.test.ts
@@ -85,6 +85,7 @@ function config(): CodexLlmProxyConfig {
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
+ imageInputModels: [],
controlApiBaseUrl: '',
controlApiServiceName: 'codex-llm-proxy',
controlApiServiceToken: '',
diff --git a/codex-llm-proxy/test/server.security.test.ts b/codex-llm-proxy/test/server.security.test.ts
index 40156c8d5..fd2ab6f55 100644
--- a/codex-llm-proxy/test/server.security.test.ts
+++ b/codex-llm-proxy/test/server.security.test.ts
@@ -1,10 +1,10 @@
-import { generateKeyPairSync } from 'node:crypto'
+import { describe, expect, it } from 'vitest'
import jwt from 'jsonwebtoken'
+import { generateKeyPairSync } from 'node:crypto'
import request from 'supertest'
-import { describe, expect, it } from 'vitest'
import { verifyAdminPermit } from '../src/auth/adminPermitVerifier.js'
import { verifyExecutionTicket } from '../src/auth/executionTicketVerifier.js'
-import { loadConfig, type CodexLlmProxyConfig } from '../src/config.js'
+import { type CodexLlmProxyConfig, loadConfig } from '../src/config.js'
import { createProxyApps } from '../src/server.js'
const { privateKey, publicKey } = generateKeyPairSync('rsa', {
@@ -24,6 +24,7 @@ function config(overrides: Partial<CodexLlmProxyConfig> = {}): CodexLlmProxyConf
jwtIssuer: 'control-api',
jwtPublicKey: publicKey,
executionEnabled: true,
+ imageInputModels: [],
controlApiBaseUrl: '',
controlApiServiceName: 'codex-llm-proxy',
controlApiServiceToken: '',
@@ -82,7 +83,9 @@ describe('codex-llm-proxy security surface', () => {
const metrics = await request(probeApp).get('/metrics')
expect(metrics.status).toBe(200)
expect(metrics.text).not.toMatch(/account|refresh|accessToken/i)
- expect((await request(runtimeApp).get('/internal/runtime/v1/codex/completions')).status).toBe(404)
+ expect((await request(runtimeApp).get('/internal/runtime/v1/codex/completions')).status).toBe(
+ 404
+ )
expect((await request(adminApp).get('/internal/admin/v1/codex/models')).status).toBe(404)
})
@@ -281,4 +284,21 @@ describe('codex-llm-proxy security surface', () => {
})
).toThrow(/bounded positive integer/)
})
+
+ it('keeps visual rollout disabled by default and refuses an incompatible envelope budget', () => {
+ expect(loadConfig({ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey }).imageInputModels).toEqual([])
+ expect(() =>
+ loadConfig({
+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ CODEX_IMAGE_INPUT_MODELS: 'gpt-5.1',
+ CODEX_LLM_PROXY_MAX_BODY_BYTES: '1024',
+ })
+ ).toThrow(/shared envelope byte budget/)
+ expect(
+ loadConfig({
+ CODEX_LLM_PROXY_JWT_PUBLIC_KEY: publicKey,
+ CODEX_IMAGE_INPUT_MODELS: ' gpt-5.1, gpt-5.3-codex ',
+ }).imageInputModels
+ ).toEqual(['gpt-5.1', 'gpt-5.3-codex'])
+ })
})
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index a223c56f9..7accde0c0 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.550",
+ "version": "0.1.551",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.550",
+ "version": "0.1.551",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 05ea1f10b..b3b94222e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.550",
+ "version": "0.1.551",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
index 3695ea73e..f87be66f6 100644
--- a/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
+++ b/control-api/src/routes/mcp-host/llmProviderAttempts.routes.ts
@@ -27,6 +27,7 @@ const ERROR_STATUS: Record<string, number> = {
host_binding_mismatch: 403,
unknown_field: 400,
invalid_request: 400,
+ payload_too_large: 413,
stale_generation: 409,
idempotency_conflict: 409,
provider_unavailable: 503,
diff --git a/control-api/src/services/llmProviderAttemptAuthorizer.ts b/control-api/src/services/llmProviderAttemptAuthorizer.ts
index 5837107f4..de09b46ba 100644
--- a/control-api/src/services/llmProviderAttemptAuthorizer.ts
+++ b/control-api/src/services/llmProviderAttemptAuthorizer.ts
@@ -1,8 +1,9 @@
import {
LIMITS,
+ buildCodexProxyEnvelope,
computeCodexPolicyHash,
- hashCodexCompletionRequestV1,
- parseCodexCompletionRequestV1,
+ hashCodexCompletionRequest,
+ parseCodexCompletionRequest,
} from '@clerum/llm-provider-attempt-contract'
import { config } from '../config.js'
import { type DbClient, pool, withTransaction } from '../db.js'
@@ -63,6 +64,7 @@ export type LlmProviderAttemptAuthorizeErrorCode =
| 'host_binding_mismatch'
| 'unknown_field'
| 'invalid_request'
+ | 'payload_too_large'
| 'stale_generation'
| 'idempotency_conflict'
| 'provider_unavailable'
@@ -204,7 +206,7 @@ export async function authorizeLlmProviderAttempt(
const caller = resolveCaller(claims)
assertClaimBinding(body, claims)
- const parsed = parseCodexCompletionRequestV1(body.request)
+ const parsed = parseCodexCompletionRequest(body.request)
if (!parsed.ok) {
throw new LlmProviderAttemptAuthorizeError('invalid_request', parsed.message)
}
@@ -246,7 +248,7 @@ export async function authorizeLlmProviderAttempt(
)
}
- const requestHash = hashCodexCompletionRequestV1(request)
+ const requestHash = hashCodexCompletionRequest(request)
if (typeof body.requestHash === 'string' && body.requestHash !== requestHash) {
throw new LlmProviderAttemptAuthorizeError(
'invalid_request',
@@ -512,6 +514,22 @@ export async function authorizeLlmProviderAttempt(
connectionRevision: connection.credentialRevision,
connectionId: connection.id,
})
+ if (request.schemaVersion === 'codex-completion-request.v2') {
+ // Measure the exact proxy envelope while the attempt, ticket and any
+ // new reservation are still transactional. Pre-redeem failures cannot
+ // use the ordinary receipt-based finalizer.
+ const envelope = buildCodexProxyEnvelope({
+ executionTicket: issued.executionTicket,
+ requestHash,
+ request,
+ })
+ if (!envelope.ok) {
+ throw new LlmProviderAttemptAuthorizeError(
+ envelope.code === 'limit' ? 'payload_too_large' : 'invalid_request',
+ envelope.message
+ )
+ }
+ }
log.info(
{
event: 'codex_attempt_authorized',
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
index a95a4d53e..7fba48636 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.test.ts
@@ -119,6 +119,16 @@ describe('POST /api/v1/mcp-host/llm/provider-attempts/authorize', () => {
.send({ request: {} })
expect(unassigned.status).toBe(403)
expect(unassigned.body).toEqual({ error: 'unassigned_connection' })
+
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockRejectedValueOnce(
+ new LlmProviderAttemptAuthorizeError('payload_too_large', 'proxy envelope exceeds limit')
+ )
+ const oversized = await request(app)
+ .post('/api/v1/mcp-host/llm/provider-attempts/authorize')
+ .set('Authorization', `Bearer ${token()}`)
+ .send({ request: {} })
+ expect(oversized.status).toBe(413)
+ expect(oversized.body).toEqual({ error: 'payload_too_large' })
})
it('returns the authorize contract without leaking tokens', async () => {
diff --git a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
index 5e28c2c9f..ae69b7ef4 100644
--- a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
@@ -1,9 +1,10 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool } from 'pg'
+import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import { initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
-import { evaluateBudgetCheck } from '../src/services/budgets/check.js'
+import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { getActiveReservation } from '../src/services/budgets/reservations.js'
import { getCodexCatalogModelState } from '../src/services/codexSubscriptionCatalog.js'
import {
@@ -54,6 +55,7 @@ function claims(): McpHostAccessClaims {
hostRefs: ['research-host'],
scope: 'workflow:approval:request',
workflowControlScopes: ['llm:codex:execute'],
+ mcpCapabilities: [],
iss: 'control-api',
aud: 'workflow-approvals',
jti: randomUUID(),
@@ -219,4 +221,126 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',
)
expect(leftover.rows[0]?.count).toBe('0')
})
+
+ it.each([false, true])(
+ 'rolls back an oversized signed V2 envelope without losing an existing reservation (presented=%s)',
+ async presented => {
+ const current = await getSafeCodexSubscriptionConnection(pool)
+ const invocationId = `visual-envelope-${randomUUID()}`
+ const request = {
+ ...REQUEST,
+ schemaVersion: 'codex-completion-request.v2',
+ messages: [{ role: 'user', content: '' }],
+ }
+ const budgetId = randomUUID()
+ const existingReservationId = randomUUID()
+ // Force the real danger-zone path without relying on rollup timing. The
+ // configured maximum task is greater than the remaining period allowance.
+ await pool.query(
+ `INSERT INTO token_budgets
+ (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
+ VALUES ($1, 'visual envelope rollback', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
+ [budgetId, JSON.stringify({ provider: ['codex-subscription'], model: [REQUEST.model] })]
+ )
+ await pool.query(
+ `INSERT INTO budget_pending_reservations
+ (id, budget_id, est_amount, task_ref, host_ref, expires_at)
+ VALUES ($1, $2, 5, 'preexisting-visual-task', 'research-host', NOW() + INTERVAL '15 minutes')`,
+ [existingReservationId, budgetId]
+ )
+ __resetBudgetCheckCache()
+ const payload = {
+ request,
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: current!.catalogRevision,
+ policyHash: computeCodexPolicyHash({
+ model: REQUEST.model,
+ catalogRevision: current!.catalogRevision,
+ credentialRevision: current!.credentialRevision,
+ connectionKey: current!.connectionKey,
+ }),
+ ...(presented ? { budgetReservationId: existingReservationId } : {}),
+ }
+ request.messages[0].content = 'x'.repeat(
+ LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(payload)) - 16
+ )
+ const counts = () =>
+ pool.query(`SELECT
+ (SELECT count(*)::text FROM llm_provider_attempts) AS attempts,
+ (SELECT count(*)::text FROM llm_provider_attempt_tickets) AS tickets,
+ (SELECT count(*)::text FROM budget_pending_reservations) AS reservations`)
+ const readExisting = () =>
+ pool.query('SELECT * FROM budget_pending_reservations WHERE id = $1', [
+ existingReservationId,
+ ])
+ const before = (await counts()).rows[0]
+ const existingBefore = (await readExisting()).rows[0]
+ let observedReservationId: string | undefined
+ let signed = false
+ try {
+ await expect(
+ authorizeLlmProviderAttempt(
+ claims(),
+ payload,
+ testDeps({
+ issueTicket: async (db, input) => {
+ observedReservationId = input.budgetReservationId
+ const active = await db.query(
+ 'SELECT id FROM budget_pending_reservations WHERE budget_id = $1',
+ [budgetId]
+ )
+ expect(active.rows).toHaveLength(presented ? 1 : 2)
+ if (presented) expect(input.budgetReservationId).toBe(existingReservationId)
+ else {
+ expect(input.budgetReservationId).not.toBe('unbudgeted')
+ expect(input.budgetReservationId).not.toBe(existingReservationId)
+ }
+ const issued = await issueRegisteredCodexExecutionTicket(db, input)
+ const registered = await db.query(
+ 'SELECT jti FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1',
+ [input.providerAttemptId]
+ )
+ expect(registered.rows).toHaveLength(1)
+ signed = true
+ return issued
+ },
+ })
+ )
+ ).rejects.toMatchObject({ code: 'payload_too_large' })
+ expect(signed).toBe(true)
+ expect((await counts()).rows[0]).toEqual(before)
+ expect((await readExisting()).rows[0]).toEqual(existingBefore)
+ if (!presented) {
+ expect(observedReservationId).toBeDefined()
+ expect(
+ (
+ await pool.query('SELECT id FROM budget_pending_reservations WHERE id = $1', [
+ observedReservationId,
+ ])
+ ).rows
+ ).toHaveLength(0)
+ }
+ const leftover = await pool.query(
+ 'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
+ [invocationId]
+ )
+ expect(leftover.rows).toHaveLength(0)
+ // A failed size check must not burn the invocation/attempt identity.
+ // Retrying that same binding with a request that fits must authorize.
+ request.messages[0].content = 'A bounded request after rollback'
+ const retried = await authorizeLlmProviderAttempt(claims(), payload, testDeps())
+ const committed = await pool.query(
+ 'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
+ [invocationId]
+ )
+ expect(committed.rows).toEqual([{ id: retried.providerAttemptId }])
+ expect((await readExisting()).rows[0]).toEqual(existingBefore)
+ } finally {
+ await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
+ __resetBudgetCheckCache()
+ }
+ }
+ )
})
diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
index 6e55f2094..6ccf08fe2 100644
--- a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
@@ -1,5 +1,14 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
-import { hashCodexCompletionRequestV1 } from '@clerum/llm-provider-attempt-contract'
+import { readFileSync } from 'node:fs'
+import { LIMITS, hashCodexCompletionRequestV1 } from '@clerum/llm-provider-attempt-contract'
+import { streamCodexCompletion } from '../../codex-llm-proxy/src/codexTransport'
+import {
+ CODEX_CATALOG_ORIGIN,
+ CODEX_COMPLETIONS_ORIGIN,
+} from '../../codex-llm-proxy/src/originPolicy'
+import { CodexLlmProxyClient } from '../../mcp-host/src/llm/codexLlmProxyClient'
+import { CodexSubscriptionProvider } from '../../mcp-host/src/llm/codexSubscription'
+import { ProviderAttemptAuthorizer } from '../../mcp-host/src/llm/providerAttemptAuthorizer'
import {
LlmProviderAttemptAuthorizeError,
type LlmProviderAttemptAuthorizerDeps,
@@ -8,6 +17,9 @@ import {
} from '../src/services/llmProviderAttemptAuthorizer.js'
import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
+// Non-operational sentinel consumed only by the injected external model fixture.
+const FIXTURE_ACCESS_VALUE = 'fixture-only'
+
const lockPluginWorkloadSdkRecipe = vi.hoisted(() => vi.fn())
const getPluginWorkloadSdkProviderAttemptForUpdate = vi.hoisted(() => vi.fn())
const pluginWorkloadSdkSpendOutcomeExists = vi.hoisted(() => vi.fn())
@@ -140,6 +152,191 @@ function deps(
}
describe('authorizeLlmProviderAttempt', () => {
+ // In-process integration, not browser E2E or live OAuth evidence. The Host,
+ // both clients, authorizer, parsers/hash and proxy projection are real.
+ // DB/grant custody and the external model are injected seams.
+ it.each([
+ ['png', 'attachment', false],
+ ['jpeg', 'attachment', true],
+ ['png', 'tool', true],
+ ['jpeg', 'tool', false],
+ ] as const)(
+ 'carries %s from %s (tools=%s) through Host authorization and proxy projection',
+ async (format, origin, withTools) => {
+ const fixtures = JSON.parse(
+ readFileSync(
+ new URL(
+ '../../packages/llm-provider-attempt-contract/fixtures/visual-requests.json',
+ import.meta.url
+ ),
+ 'utf8'
+ )
+ )
+ let authorized: Awaited<ReturnType<typeof authorizeLlmProviderAttempt>> | undefined
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: 'http://test-control/authorize',
+ readPlatformJwt: () => FIXTURE_ACCESS_VALUE,
+ fetchFn: async (_url, init) => {
+ authorized = await authorizeLlmProviderAttempt(
+ claims(),
+ JSON.parse(String(init?.body)),
+ deps()
+ )
+ return Response.json(authorized)
+ },
+ })
+ let projectedBody: Record<string, unknown> | undefined
+ const finalize = vi.fn(async () => ({
+ providerAttemptId: authorized!.providerAttemptId,
+ outcome: 'success' as const,
+ duplicate: false,
+ }))
+ const proxy = new CodexLlmProxyClient({
+ runtimeUrl: 'http://test-proxy/completions',
+ readPlatformJwt: () => FIXTURE_ACCESS_VALUE,
+ fetchFn: async (_url, init) => {
+ expect(authorized).toBeDefined()
+ const envelope = JSON.parse(String(init?.body))
+ const frames: unknown[] = []
+ const result = await streamCodexCompletion({
+ ...envelope,
+ imageInputEnabled: true,
+ ticket: {
+ jti: 'fixture-ticket',
+ hostRef: 'research-host',
+ model: REQUEST.model,
+ requestHash: authorized!.requestHash,
+ providerAttemptId: authorized!.providerAttemptId,
+ },
+ redeem: async () => ({
+ accessToken: FIXTURE_ACCESS_VALUE,
+ chatgptAccountId: 'fixture-account',
+ attemptReceipt: 'a'.repeat(64),
+ expiryClass: 'short_lived',
+ transport: {
+ protocolVersion: 'codex-subscription-transport.v1',
+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,
+ catalogOrigin: CODEX_CATALOG_ORIGIN,
+ operation: 'completion_stream',
+ servedModel: REQUEST.model,
+ maxStreamDurationMs: 1000,
+ },
+ }),
+ finalize,
+ lookup: async () => [{ address: '1.2.3.4', family: 4 }],
+ fetchFn: async (_upstream, requestInit) => {
+ projectedBody = JSON.parse(String(requestInit?.body))
+ const parts = (
+ projectedBody!.input as Array<{ role?: string; content: Array<{ type: string }> }>
+ ).find(item => item.role === 'user')!.content
+ expect(parts.some(part => part.type === 'input_image')).toBe(true)
+ return new Response(
+ 'data: {"type":"response.output_text.delta","delta":"image received"}\n\ndata: {"type":"response.completed"}\n\n'
+ )
+ },
+ onFrame: frame => frames.push(frame),
+ })
+ frames.push({ type: 'done', ...result })
+ return new Response(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(''))
+ },
+ })
+ const provider = new CodexSubscriptionProvider(REQUEST.model, {
+ authorizer,
+ proxy,
+ imageInputEnabled: true,
+ attemptContext: () => ({
+ policyRevision: 4,
+ policyHash: body().policyHash,
+ hostRef: 'research-host',
+ }),
+ })
+ const image = fixtures[format].messages[0].contentParts.find(
+ (part: { type: string }) => part.type === 'image'
+ )
+ let messages = fixtures[format].messages
+ if (origin === 'tool') {
+ image.source = { kind: 'tool', attachmentId: 'fixture-image', toolCallId: 'fixture-call' }
+ messages = [
+ {
+ role: 'assistant',
+ content: '',
+ tool_calls: [{ id: 'fixture-call', name: 'screenshot', arguments: {} }],
+ },
+ {
+ role: 'tool',
+ content: 'Screenshot ready',
+ tool_call_id: 'fixture-call',
+ name: 'screenshot',
+ },
+ ...messages,
+ ]
+ }
+ const result = withTools
+ ? await provider.completeSingleTurnWithTools(messages, [
+ {
+ name: 'screenshot',
+ description: 'Take a screenshot',
+ parameters: { type: 'object', properties: {} },
+ },
+ ])
+ : await provider.completeSingleTurn(messages)
+ expect(
+ (projectedBody!.input as Array<{ role?: string; content: unknown[] }>).find(
+ item => item.role === 'user'
+ )!.content
+ ).toContainEqual({
+ type: 'input_image',
+ image_url: `data:${image.mimeType};base64,${image.data}`,
+ })
+ if (origin === 'tool') {
+ expect((projectedBody!.input as Array<{ type?: string }>).map(item => item.type)).toEqual([
+ 'function_call',
+ 'function_call_output',
+ undefined,
+ ])
+ }
+ expect(result.content).toBe('image received')
+ expect(finalize).toHaveBeenCalledOnce()
+ }
+ )
+
+ it('checks the V2 proxy envelope inside the transaction, after signing and before commit', async () => {
+ const transactionEvents: string[] = []
+ const request = {
+ ...REQUEST,
+ schemaVersion: 'codex-completion-request.v2',
+ messages: [{ role: 'user', content: '' }],
+ }
+ const payload = body({ request })
+ request.messages[0].content = 'x'.repeat(
+ LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(payload)) - 16
+ )
+ expect(Buffer.byteLength(JSON.stringify(payload))).toBeLessThan(LIMITS.maxRequestBodyBytes)
+ const current = deps({
+ withTransaction: async work => {
+ transactionEvents.push('begin')
+ try {
+ const result = await work({ query: vi.fn() } as never)
+ transactionEvents.push('commit')
+ return result
+ } catch (error) {
+ transactionEvents.push('rollback')
+ throw error
+ }
+ },
+ // A bounded synthetic signed envelope contribution; no live account material.
+ issueTicket: vi.fn().mockResolvedValue({
+ executionTicket: 't'.repeat(4096),
+ expiresAt: new Date('2026-09-16T23:00:00Z'),
+ }),
+ })
+ await expect(authorizeLlmProviderAttempt(claims(), payload, current)).rejects.toMatchObject({
+ code: 'payload_too_large',
+ })
+ expect(current.issueTicket).toHaveBeenCalledOnce()
+ expect(transactionEvents).toEqual(['begin', 'rollback'])
+ })
+
let current: LlmProviderAttemptAuthorizerDeps
beforeEach(() => {
diff --git a/desktop-app/test/e2e-playwright/codex-image-input.spec.ts b/desktop-app/test/e2e-playwright/codex-image-input.spec.ts
new file mode 100644
index 000000000..db129fe7e
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/codex-image-input.spec.ts
@@ -0,0 +1,145 @@
+/**
+ * E2E_GUARDIAN_IPC_FLOW: Desktop sends through main-process IPC. Observe the
+ * real chooser, preview, user message, settled composer and visual-only answer.
+ * Real upstream lane, never evidence when only statically checked.
+ * Login fixtures provision the test identity and sign in through the UI. Their
+ * existing stored-session reset needs explicit opt-in before fixtures run.
+ */
+import { randomUUID } from 'node:crypto'
+import { challengeImage } from './codexImageChallenge.js'
+import { expect, test } from './fixtures.js'
+
+const hostRef = process.env.E2E_HOST_REF ?? ''
+const hostLabel = process.env.E2E_CODEX_HOST_LABEL ?? ''
+const modelId = process.env.E2E_CODEX_IMAGE_MODEL ?? ''
+const modelLabel = process.env.E2E_CODEX_IMAGE_MODEL_LABEL ?? ''
+const mode = process.env.E2E_CODEX_IMAGE_MODE ?? ''
+const escapeRegex = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+test.beforeAll(() => {
+ expect(
+ process.env.E2E_CODEX_IMAGE_INPUT,
+ 'Use the dedicated, explicitly authorized image lane'
+ ).toBe('1')
+ expect(process.env.CODEX_REAL_UPSTREAM_CONFIRM, 'Explicit upstream authorization required').toBe(
+ '1'
+ )
+ expect(
+ process.env.E2E_CODEX_ALLOW_SESSION_RESET,
+ 'The existing login fixture resets its stored session'
+ ).toBe('1')
+ expect(hostRef, 'Name the owned Host explicitly').not.toBe('')
+ expect(hostLabel, 'Name its visible Agents-list label explicitly').not.toBe('')
+ expect(modelId, 'Name the actual Codex model explicitly').not.toBe('')
+ expect(modelLabel, 'Name its visible model-picker label explicitly').not.toBe('')
+ expect(['enabled', 'disabled'], 'Select a provisioned capability lane explicitly').toContain(mode)
+})
+
+async function startOwnedChat(page: import('@playwright/test').Page): Promise<void> {
+ const agents = page.getByTestId('nav-agents')
+ // Snapshot only chooses the navigation layout; subsequent assertions retry.
+ if (!(await agents.isVisible())) {
+ const settings = page.getByTestId('nav-settings-menu')
+ await expect(settings).toBeVisible()
+ if ((await settings.getAttribute('aria-expanded')) !== 'true') await settings.click()
+ await expect(settings).toHaveAttribute('aria-expanded', 'true')
+ }
+ await expect(agents).toBeVisible()
+ await agents.click()
+ const actions = page.getByRole('button', { name: `More actions for ${hostLabel}`, exact: true })
+ await expect(actions).toBeVisible({ timeout: 30_000 })
+ await actions.click()
+ await expect(actions).toHaveAttribute('aria-expanded', 'true')
+ const actionsMenu = page.getByRole('menu').filter({
+ has: page.getByRole('button', { name: 'New chat', exact: true }),
+ })
+ await expect(actionsMenu).toBeVisible()
+ const newChat = actionsMenu.getByRole('button', { name: 'New chat', exact: true })
+ await expect(newChat).toBeVisible()
+ await newChat.click()
+ await expect(page.getByTestId('chat-input')).toBeVisible()
+ await expect(page.getByTestId('agent-response')).toHaveCount(0)
+ const selector = page.getByTestId('model-selector-up')
+ await expect(selector).toHaveAttribute('data-host-ref', hostRef, { timeout: 30_000 })
+ await expect(selector).toHaveAttribute('data-provider', 'codex-subscription')
+ await selector.getByRole('button', { name: /^Model — / }).click()
+ const menu = selector.getByRole('menu', { name: 'Select model', exact: true })
+ await expect(menu).toBeVisible()
+ await menu
+ .getByRole('menuitemradio', {
+ name: new RegExp(`^${escapeRegex(modelLabel)}(?:\\s*default)?$`),
+ })
+ .click()
+ await expect(selector).toHaveAttribute('data-model', modelId, { timeout: 30_000 })
+}
+
+for (const format of ['png', 'jpeg'] as const) {
+ test(`${format} visual input follows the explicitly provisioned capability lane`, async ({
+ appPage,
+ }) => {
+ test.setTimeout(240_000)
+ // The random answer exists only in pixels, never the filename or prompt.
+ // A generic response or discarded image cannot satisfy this 64-bit challenge.
+ const image = challengeImage(format)
+ const filename = `visual-${randomUUID()}.${format}`
+ const prompt = 'Read the hexadecimal code in the attached image. Reply with that code only.'
+ await test.step('select the owned Codex Host and model through UI', async () => {
+ await startOwnedChat(appPage)
+ })
+ await test.step('upload through the chooser and verify the rendered preview', async () => {
+ await appPage.getByRole('button', { name: 'Add context', exact: true }).click()
+ const upload = appPage.getByRole('menuitem', { name: 'Upload Files', exact: true })
+ await expect(upload).toBeVisible()
+ const chooserReady = appPage.waitForEvent('filechooser')
+ await upload.click()
+ const chooser = await chooserReady
+ await chooser.setFiles({ name: filename, mimeType: `image/${format}`, buffer: image.bytes })
+ const chip = appPage.getByRole('button', { name: filename, exact: true })
+ await expect(chip).toBeVisible()
+ await chip.click()
+ const preview = appPage.getByRole('dialog', { name: filename, exact: true })
+ await expect(preview).toBeVisible()
+ await expect(preview.getByRole('img', { name: filename, exact: true })).toBeVisible()
+ await preview.getByRole('button', { name: 'Close image preview', exact: true }).click()
+ await expect(preview).toHaveCount(0)
+ })
+ await test.step('submit and assert the correlated terminal result', async () => {
+ const composer = appPage.getByTestId('chat-input')
+ await composer.fill(prompt)
+ const send = appPage.getByTestId('send-button')
+ await expect(send).toBeEnabled()
+ await send.click()
+ await expect(composer).toHaveValue('')
+ const thread = appPage.getByTestId('message-list')
+ await expect(thread.getByText(prompt, { exact: true })).toBeVisible()
+ await expect(
+ thread.locator('.message-attachment-label').filter({ hasText: filename })
+ ).toBeVisible()
+ const answer = appPage.getByTestId('agent-response')
+ const expectedCode = new RegExp(`\\b${image.code}\\b`, 'i')
+ if (mode === 'enabled') {
+ // One terminal wait, not separate 180-second waits for creation and text.
+ await expect(answer).toContainText(expectedCode, { timeout: 180_000 })
+ await expect(answer).not.toHaveClass(/chat-bubble--error/)
+ // Do not let tool-based attachment loading/OCR substitute for the
+ // direct model-input behavior this lane must prove.
+ await expect(thread.getByTestId('progress-stepper')).toHaveCount(0)
+ } else {
+ await expect(answer.locator('.error-bubble-message')).toContainText(
+ 'Image input is not enabled for this Codex model',
+ { timeout: 180_000 }
+ )
+ await expect(answer).toHaveClass(/chat-bubble--error/)
+ await expect(answer).not.toContainText(expectedCode)
+ }
+ await expect(answer).toHaveCount(1)
+ await expect(send).toHaveAttribute('aria-label', 'Send message')
+ await expect(thread.locator('.chat-message--in-flight')).toHaveCount(0)
+ await expect(appPage.getByRole('status', { name: /^Running on fallback:/ })).toHaveCount(0)
+ const selector = appPage.getByTestId('model-selector-up')
+ await expect(selector).toHaveAttribute('data-host-ref', hostRef)
+ await expect(selector).toHaveAttribute('data-provider', 'codex-subscription')
+ await expect(selector).toHaveAttribute('data-model', modelId)
+ })
+ })
+}
diff --git a/desktop-app/test/e2e-playwright/codexImageChallenge.ts b/desktop-app/test/e2e-playwright/codexImageChallenge.ts
new file mode 100644
index 000000000..daa6e034c
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/codexImageChallenge.ts
@@ -0,0 +1,23 @@
+import { randomBytes } from 'node:crypto'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+
+/** A neutral image with an answer absent from the filename and user prompt. */
+export function challengeImage(format: 'png' | 'jpeg'): { code: string; bytes: Buffer } {
+ // Reuse the Host renderer installed for T0; do not add another dependency.
+ const { createCanvas } = createRequire(path.resolve(__dirname, '../../../mcp-host/package.json'))(
+ '@napi-rs/canvas'
+ )
+ const code = randomBytes(8).toString('hex').toUpperCase()
+ const canvas = createCanvas(800, 120)
+ const context = canvas.getContext('2d')
+ context.fillStyle = 'white'
+ context.fillRect(0, 0, 800, 120)
+ context.fillStyle = 'black'
+ context.font = '48px monospace'
+ context.fillText(code, 24, 78)
+ return {
+ code,
+ bytes: format === 'png' ? canvas.toBuffer('image/png') : canvas.toBuffer('image/jpeg'),
+ }
+}
diff --git a/desktop-app/test/e2e-playwright/playwright.codex-image.config.ts b/desktop-app/test/e2e-playwright/playwright.codex-image.config.ts
new file mode 100644
index 000000000..969dbca27
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/playwright.codex-image.config.ts
@@ -0,0 +1,28 @@
+import { defineConfig } from '@playwright/test'
+import base from './playwright.config.js'
+
+// Validate admission before the inherited global setup can contact services.
+// A missing opt-in is a failed explicit lane, never a successful all-skipped run.
+for (const name of [
+ 'E2E_CODEX_IMAGE_INPUT',
+ 'CODEX_REAL_UPSTREAM_CONFIRM',
+ 'E2E_CODEX_ALLOW_SESSION_RESET',
+]) {
+ if (process.env[name] !== '1') throw new Error(`Codex image lane requires ${name}=1`)
+}
+for (const name of [
+ 'E2E_HOST_REF',
+ 'E2E_CODEX_HOST_LABEL',
+ 'E2E_CODEX_IMAGE_MODEL',
+ 'E2E_CODEX_IMAGE_MODEL_LABEL',
+]) {
+ if (!process.env[name]?.trim()) throw new Error(`Codex image lane requires explicit ${name}`)
+}
+if (!['enabled', 'disabled'].includes(process.env.E2E_CODEX_IMAGE_MODE ?? '')) {
+ throw new Error('Codex image lane requires E2E_CODEX_IMAGE_MODE=enabled or disabled')
+}
+
+export default defineConfig({
+ ...base,
+ projects: [{ name: 'codex-image-input', testMatch: '**/codex-image-input.spec.ts' }],
+})
diff --git a/desktop-app/test/e2e-playwright/playwright.config.ts b/desktop-app/test/e2e-playwright/playwright.config.ts
index ec0b57fa8..569b56cd2 100644
--- a/desktop-app/test/e2e-playwright/playwright.config.ts
+++ b/desktop-app/test/e2e-playwright/playwright.config.ts
@@ -163,7 +163,7 @@ export default defineConfig({
projects: [
{
name: 'default',
- testIgnore: '**/gfs-upload-v2.test.ts',
+ testIgnore: ['**/gfs-upload-v2.test.ts', '**/codex-image-input.spec.ts'],
},
{
name: 'packaged-gfs-upload-v2',
diff --git a/desktop-app/ui/src/components/agents/ModelSelector.tsx b/desktop-app/ui/src/components/agents/ModelSelector.tsx
index 28e23585b..1a35d0fc5 100644
--- a/desktop-app/ui/src/components/agents/ModelSelector.tsx
+++ b/desktop-app/ui/src/components/agents/ModelSelector.tsx
@@ -132,7 +132,13 @@ export function ModelSelector({ agentRef, chatId, placement = 'down' }: ModelSel
// Render a static, non-interactive chip with an explanatory tooltip.
if (data.degraded) {
return (
- <div className={rootClassName}>
+ <div
+ className={rootClassName}
+ data-testid={`model-selector-${placement}`}
+ data-host-ref={agentRef}
+ data-provider={data.provider}
+ data-model={effectiveModel}
+ >
<Pill
tone="neutral"
size="sm"
@@ -148,7 +154,14 @@ export function ModelSelector({ agentRef, chatId, placement = 'down' }: ModelSel
}
return (
- <div className={rootClassName} ref={containerRef}>
+ <div
+ className={rootClassName}
+ ref={containerRef}
+ data-testid={`model-selector-${placement}`}
+ data-host-ref={agentRef}
+ data-provider={data.provider}
+ data-model={effectiveModel}
+ >
<Pill
tone={data.sessionModelBlocked ? 'warning' : 'neutral'}
size="sm"
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index 5ce49ad09..62b15867b 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -2,6 +2,76 @@
Frozen protocol version: `codex-subscription-transport.v1`.
+Request schemas: `codex-completion-request.v1` for ordinary text and
+`codex-completion-request.v2` for ordered visual content (including text parts
+left after media pruning). The transport, execution-ticket type and receipt
+version remain V1. Request V2 does not imply a transport protocol upgrade.
+
+## Visual requests (issue #650)
+
+V2 user messages may carry `contentParts`: ordered text parts or inline PNG/JPEG
+parts. Images contain `mimeType`, canonical base64 `data`, and a closed `source`
+identity: `{ kind: 'attachment', attachmentId, messageId }` or
+`{ kind: 'tool', attachmentId, toolCallId }`. Sources are established by the Host
+message/tool producers, included in the canonical request hash, and omitted from
+the upstream projection. They are attribution inside the authorized request,
+not permission to fetch another object. No arbitrary URL is accepted.
+
+The text parts joined with a newline must equal the message's `content`.
+Producers synchronize the fields when adding turn context. After media pruning,
+the remaining text parts are authoritative, so redaction text is preserved.
+Ordinary V1 textual requests retain their existing wire representation and hashes.
+
+Tool-image source retention is enabled only when a configured primary or fallback
+provider requires source identity. Pure API-key chains keep the existing loop
+deduplication and message shape. Mixed chains retain distinct source identities
+in canonical history; each provider adapter selects its own view without mutating
+that history. The internal `sourceIdentityOnly` marker identifies extra copies and
+never enters the Codex request schema. API-key adapters omit those copies when an
+unmarked representative remains. If pruning removed that representative, they
+retain one remaining image per MIME/bytes pair and its explanatory text. They do
+not restore pixels removed by pruning. The user-facing attachment collection
+keeps its existing deduplication behavior.
+
+Local visual budgets are three images, 512 KiB decoded per image, 640 KiB decoded
+across a request, 8192 pixels per dimension and 16,777,216 pixels per image.
+These are conservative Evenfire limits, not claims about upstream capabilities.
+The shared pure validator checks canonical base64, MIME/container framing and
+header dimensions; it does not decode pixels or prove image decodability.
+The fixtures contain independently decoded 2x2 PNG/JPEG images.
+
+The existing 1 MiB bound still applies to each complete serialized request and
+HTTP envelope, including text, tools, history and the signed execution ticket.
+The authorizer builds the exact V2 proxy envelope inside its transaction after
+signing but before commit. Exceeding the bound rolls back the new attempt,
+ticket and new reservation; it does not call the receipt finalizer before redeem.
+The Host uses the same envelope builder. V2 has no outer deadline: its deadline
+is `request.deadlineMs`, part of the authorized hash. A proxy configured below
+the shared envelope budget refuses visual activation at startup.
+
+The proxy projects parts to Responses `input_text` and `input_image` items with
+an inline data URL. The shape is grounded in the official Codex client
+[ContentItem and ImageReference definitions](https://github.com/openai/codex/blob/fc2ea82e7eff22c618a56db29c68a6b1967cba7d/codex-rs/protocol/src/models.rs#L878).
+This source evidence is not a successful call to Evenfire's frozen endpoint.
+
+`CODEX_IMAGE_INPUT_MODELS` is an explicit, comma-separated model rollout gate in
+both Host and proxy. It defaults to empty. Set it only for the models whose
+interoperability has been independently verified, after every authorizer/proxy
+replica accepts V2. This gate is not a replacement for catalog authorization.
+Until enabled, visual input returns `image_input_unsupported`; it is not silently
+removed and does not trigger automatic fallback. Ordinary eligible failures
+retain the existing configured fallback policy. Limit errors, including HTTP
+413 without a JSON body, remain non-retryable.
+
+Deploy accepting consumers before visual senders. An old consumer must reject
+V2 explicitly; do not translate V2 to text to work around that rejection. On
+rollback disable visual activation first, drain attempts, then restore consumers.
+Do not roll back to a sender that silently discards images without a front guard.
+Committed abandoned attempts retain their audit identity. Existing ticket and
+budget TTLs, rather than the receipt finalizer, bound their execution and pending
+budget; a new physical attempt must have a fresh attempt binding. Tests of these
+lifecycle guarantees and a real upstream image check are separate acceptance lanes.
+
This document is the Phase 0 architecture freeze for provider `codex-subscription`.
It is not a runtime client and does not authorize API-key billing.
diff --git a/docs/testing/codex-subscription-validation.md b/docs/testing/codex-subscription-validation.md
index 312ab128b..84d70c011 100644
--- a/docs/testing/codex-subscription-validation.md
+++ b/docs/testing/codex-subscription-validation.md
@@ -2,6 +2,78 @@
Evidence lanes for `codex-subscription`. One lane does not stand in for another.
+## Image input validation (#650)
+
+The shared package tests V1 golden-hash stability and V2 parsing, source identity,
+ordered text/image content and local image/envelope budgets. Its
+`fixtures/visual-requests.json` contains real 2x2 PNG/JPEG samples;
+`fixtures/canonical-request-hashes.v2.json` freezes the new projection separately.
+The image parser validates container structure and dimensions, not decoded pixels.
+
+Provider tests must cover both completion methods, direct/tool origins, prompt
+context, redacted text parts, repeated bytes from distinct tool calls and the
+default-disabled model gate. Proxy conformance inspects the final upstream body,
+asserts provenance does not become model input, and retains tools/cancel/SSE
+checks. Authorizer tests require a pre-commit exact-envelope check; its PostgreSQL
+case must prove rollback against a real transaction. A mocked transaction result
+does not satisfy that PostgreSQL gate.
+
+Set `CODEX_IMAGE_INPUT_MODELS` consistently on the owned Host/proxy only after
+consumer rollout and a separately authorized neutral-image interoperability check.
+The default empty list rejects visual input clearly. No test fixture or passing
+conformance suite certifies that the real endpoint or selected model sees images.
+
+Keep browser/Electron evidence separate: visible login and Host selection, upload
+through Add context / Upload Files, preview, send, correlated task completion and
+visible answer. Assert a business signal tied to the submitted image and current
+task; an answer saying it saw an image is insufficient. Do not use SDK text-only
+journeys or GFS file upload as substitutes for composer image input. Run the E2E
+static auditor on changed browser specs and record external prerequisites rather
+than silently skipping them.
+
+### Opt-in Desktop image lane
+
+`desktop-app/test/e2e-playwright/codex-image-input.spec.ts` covers direct PNG/JPEG
+uploads through the visible composer. It creates a fresh 64-bit hexadecimal
+challenge rendered only into image pixels. The filename and prompt do not carry
+the answer. An enabled run requires that answer (hexadecimal case is ignored), a non-error response, no
+tool steps substituting OCR/loading for direct image input, and no fallback badge.
+The selector's stable data attributes verify the actual selected Host, provider
+and model. The disabled mode instead requires the explicit capability error.
+
+This is a real upstream lane, not a mock. Before running it, obtain separate
+authorization for runtime, upstream access and the existing login fixture's
+stored-session reset. Provision the owned Host and consumers first. Set:
+
+- `E2E_CODEX_IMAGE_INPUT=1` and `CODEX_REAL_UPSTREAM_CONFIRM=1` only for that
+ authorized run; `E2E_CODEX_ALLOW_SESSION_RESET=1` acknowledges the fixture's
+ session reset.
+- `E2E_HOST_REF` to the actual owned Host and `E2E_CODEX_HOST_LABEL` to its
+ exact visible label in the Agents list; there is no shared-host default.
+- `E2E_CODEX_IMAGE_MODEL` and `E2E_CODEX_IMAGE_MODEL_LABEL` to its model ID and
+ visible picker label.
+- `E2E_CODEX_IMAGE_MODE=enabled` or `disabled`, matching the provisioned
+ capability gate. Run both modes against their appropriate configuration;
+ the test does not change deployments or grants.
+
+Install Host dependencies as well as Desktop dependencies: the neutral challenge
+uses the existing Host canvas dependency. Node 24 and `verify:electron` remain
+mandatory. After loading the approved branch-owned service URLs and prerequisites,
+the command from `desktop-app` is:
+
+```bash
+node node_modules/@playwright/test/cli.js test \
+ --config test/e2e-playwright/playwright.codex-image.config.ts --project codex-image-input
+```
+
+The normal Desktop project excludes this spec. Its dedicated config rejects
+missing authority, Host/model identity or mode before global setup; it cannot
+return a successful all-skipped run. The model ID must be the canonical ID
+returned by the runtime, not an alias. Static audit/typecheck and local challenge-image
+decoding do not count as browser or real-upstream execution. Typed tool-result
+images remain a separate integration/runtime journey; this direct-input test
+does not certify GFS producers or retained image history.
+
## Lanes
- **T0** — unit, schema, hash, contracts, typecheck, lint, build. The
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 4bc42dd59..859359182 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.337",
+ "version": "0.1.338",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.337",
+ "version": "0.1.338",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index ca60516eb..74e253d43 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.337",
+ "version": "0.1.338",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/agent/__tests__/taskExecutor.test.ts b/mcp-host/src/agent/__tests__/taskExecutor.test.ts
index b7a617a22..209ba9ae6 100644
--- a/mcp-host/src/agent/__tests__/taskExecutor.test.ts
+++ b/mcp-host/src/agent/__tests__/taskExecutor.test.ts
@@ -15,6 +15,8 @@ import type { Attachment, ChatMessage, MessageContentPart, TraceContextV1 } from
import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
import { anthropicApiError } from '../../llm/__tests__/sdkErrorFixtures'
import { ClaudeProvider } from '../../llm/claude'
+import { FailoverEngine } from '../../llm/failover/engine'
+import type { LlmPolicy } from '../../llm/failover/types'
import { PromptCache } from '../../llm/promptCache'
import { logger } from '../../logger'
import type { Task, TaskError, TaskSource } from '../../queue/types'
@@ -988,7 +990,7 @@ describe('TaskExecutor', () => {
expect(deps.onComplete).toHaveBeenCalledTimes(1)
})
- it.each(['openai', 'claude', 'zai', 'bailian'] as const)(
+ it.each(['openai', 'claude', 'zai', 'bailian', 'codex-subscription'] as const)(
'injects text+image contentParts for %s provider',
async providerType => {
vi.mocked(runToolUseLoop).mockResolvedValueOnce({ type: 'response', content: 'ok' } as any)
@@ -1006,14 +1008,21 @@ describe('TaskExecutor', () => {
await executor.run()
const userMessage = getLastUserMessageFromLoopCall()
- expect(userMessage.contentParts).toEqual([
- { type: 'text', text: 'Analyze this image' },
- {
- type: 'image',
- mimeType: 'image/jpeg',
- data: 'ZmFrZS1pbWFnZS1iYXNlNjQ=',
- },
- ] satisfies MessageContentPart[])
+ expect(vi.mocked(runToolUseLoop).mock.calls[0][0].imageSourceIdentity).toBe(
+ providerType === 'codex-subscription'
+ )
+ const parts = userMessage.contentParts ?? []
+ expect(parts).toHaveLength(2)
+ // The prompt-cache turn-context block rides with the text part, so
+ // `content` and its text parts stay equal for the Codex V2 contract.
+ expect(parts[0]).toEqual({ type: 'text', text: userMessage.content })
+ expect(userMessage.content.endsWith('Analyze this image')).toBe(true)
+ expect(parts[1]).toEqual({
+ type: 'image',
+ mimeType: 'image/jpeg',
+ data: 'ZmFrZS1pbWFnZS1iYXNlNjQ=',
+ source: { kind: 'attachment', attachmentId: 'att-1', messageId: 'msg-1' },
+ } satisfies MessageContentPart)
}
)
@@ -1029,14 +1038,74 @@ describe('TaskExecutor', () => {
await executor.run()
const userMessage = getLastUserMessageFromLoopCall()
- expect(userMessage.contentParts).toEqual([
- { type: 'text', text: 'User attached image(s).' },
- {
- type: 'image',
- mimeType: 'image/png',
- data: 'cG5n',
- },
- ] satisfies MessageContentPart[])
+ const parts = userMessage.contentParts ?? []
+ expect(parts).toHaveLength(2)
+ expect(parts[0]).toEqual({ type: 'text', text: userMessage.content })
+ expect(userMessage.content.endsWith('User attached image(s).')).toBe(true)
+ expect(parts[1]).toEqual({
+ type: 'image',
+ mimeType: 'image/png',
+ data: 'cG5n',
+ source: { kind: 'attachment', attachmentId: 'att-1', messageId: 'msg-1' },
+ } satisfies MessageContentPart)
+ })
+
+ it.each(['claude', 'codex-subscription'])(
+ 'selects image identity from the configured fallback %s',
+ async fallback => {
+ vi.mocked(runToolUseLoop).mockResolvedValueOnce({ type: 'response', content: 'ok' } as any)
+ const policy: LlmPolicy = {
+ fallbacks: [{ provider: fallback, model: 'fallback-model' }],
+ triggerOn: ['provider_unavailable'],
+ cooldownSeconds: 30,
+ }
+ const deps = createDeps({
+ failover: { policy, engine: new FailoverEngine(policy), buildProvider: () => null },
+ })
+ await new TaskExecutor(createTask('hello'), deps).run()
+ expect(vi.mocked(runToolUseLoop).mock.calls[0][0].imageSourceIdentity).toBe(
+ fallback === 'codex-subscription'
+ )
+ }
+ )
+
+ it('preserves history enrichment when adding image parts', async () => {
+ vi.mocked(runToolUseLoop).mockResolvedValueOnce({ type: 'response', content: 'ok' } as any)
+ const deps = createDeps()
+ const original = deps.conversationManager.buildMessageHistory.bind(deps.conversationManager)
+ vi.spyOn(deps.conversationManager, 'buildMessageHistory').mockImplementation(conversation => {
+ const messages = original(conversation)
+ const last = messages[messages.length - 1]
+ if (last?.role === 'user') last.content = `Conversation context: ${last.content}`
+ return messages
+ })
+ const task = createTask('Analyze this image')
+ task.sourceMessage!.attachments = [createImageAttachment()]
+ await new TaskExecutor(task, deps).run()
+ const message = getLastUserMessageFromLoopCall()
+ expect(message.content).toContain('Conversation context: Analyze this image')
+ expect(message.contentParts?.[0]).toEqual({ type: 'text', text: message.content })
+ })
+
+ it('uses the queued task id when the source message carries no delivery id', async () => {
+ vi.mocked(runToolUseLoop).mockResolvedValueOnce({ type: 'response', content: 'ok' } as any)
+ const deps = createDeps()
+ const task = createTask('Analyze this image')
+ task.sourceMessage!.messageId = ' '
+ task.sourceMessage!.attachments = [createImageAttachment()]
+
+ const executor = new TaskExecutor(task, deps)
+ await executor.run()
+
+ const parts = getLastUserMessageFromLoopCall().contentParts ?? []
+ expect(parts[1]).toEqual({
+ type: 'image',
+ mimeType: 'image/jpeg',
+ data: 'ZmFrZS1pbWFnZS1iYXNlNjQ=',
+ // A real identity, not a placeholder: internal sources have no delivery
+ // id, so the task that queued the message identifies it.
+ source: { kind: 'attachment', attachmentId: 'att-1', messageId: task.id },
+ } satisfies MessageContentPart)
})
it('skips contentParts when provider is unsupported', async () => {
diff --git a/mcp-host/src/agent/taskExecutor.ts b/mcp-host/src/agent/taskExecutor.ts
index 79d2f1c2d..4b549c5cc 100644
--- a/mcp-host/src/agent/taskExecutor.ts
+++ b/mcp-host/src/agent/taskExecutor.ts
@@ -85,12 +85,13 @@ import type {
ToolResult,
} from '../core/types'
import { ApprovalExpiredError } from '../core/types'
+import { prependTextToParts, textContentFromParts } from '../core/types'
import type { UsageContext } from '../core/types'
import type { TaskLifecycle } from '../lifecycle/taskLifecycle'
import type { SingleTurnProvider } from '../llm'
import type { PromptCache } from '../llm/promptCache'
import { stampStableHashGauge } from '../llm/promptCacheMetrics'
-import { isLlmProvider } from '../llm/registryCore'
+import { descriptorFor, isLlmProvider } from '../llm/registryCore'
import { logger } from '../logger'
import type { McpManager } from '../mcp'
import { getDisplayName, sanitizeError } from '../progress/intentExtraction.js'
@@ -1008,12 +1009,17 @@ export class TaskExecutor {
// History rehydration + pre-prune compaction are part of the session-load
// cost the user pays before the first model byte.
this.turnTiming?.addSessionLoadMs(Date.now() - historyStart)
- const sourceMessageAttachments = this.buildSourceMessageContentParts()
+ const lastMessage = messages[messages.length - 1]
+ const sourceMessageAttachments = this.buildSourceMessageContentParts(
+ lastMessage?.role === 'user' ? lastMessage.content : ''
+ )
if (sourceMessageAttachments.length > 0) {
- const lastMessage = messages[messages.length - 1]
if (lastMessage?.role === 'user') {
messages[messages.length - 1] = {
...lastMessage,
+ // Parts are authoritative: `content` is restated from the text parts
+ // so a message with images cannot contradict its own projection.
+ content: textContentFromParts(sourceMessageAttachments),
contentParts: sourceMessageAttachments,
}
}
@@ -1066,6 +1072,13 @@ export class TaskExecutor {
: undefined,
})
const isCron = this.task.cronJobId !== undefined
+ if (m.contentParts && m.contentParts.length > 0) {
+ // Keep the parts/`content` invariant the Codex V2 parser enforces: the
+ // block moves with the text it is prepended to.
+ const contentParts = prependTextToParts(m.contentParts, block)
+ messages[i] = { ...m, contentParts, content: textContentFromParts(contentParts) }
+ return
+ }
const baseContent =
m.content && m.content.length > 0 ? m.content : isCron ? '<cron task>' : m.content
messages[i] = { ...m, content: block + baseContent }
@@ -1448,6 +1461,15 @@ export class TaskExecutor {
toolProgressInterval: appConfig.nativeTool.toolProgressInterval,
})
loopConfig.abortSignal = this.abortController.signal
+ loopConfig.imageSourceIdentity =
+ this.deps.llmProvider.requiresImageSourceIdentity === true ||
+ [
+ this.deps.llmProvider.getProviderType(),
+ ...(this.deps.failover?.policy.fallbacks.map(entry => entry.provider) ?? []),
+ ].some(
+ provider =>
+ isLlmProvider(provider) && descriptorFor(provider).requiresImageSourceIdentity === true
+ )
loopConfig.onAttachments = attachments =>
mergeCollectedAttachments(this.completedAttachments, attachments)
// Guardrails (spec §6) — build the tool-lane guardrail from the Host block.
@@ -2103,7 +2125,7 @@ export class TaskExecutor {
return attachments
}
- private buildSourceMessageContentParts(): MessageContentPart[] {
+ private buildSourceMessageContentParts(messageContent: string): MessageContentPart[] {
const providerType = this.deps.llmProvider.getProviderType()
// getProviderType() always returns a registered LlmProvider, so in practice
// this never drops images for a known provider — that's the point: a new
@@ -2116,6 +2138,12 @@ export class TaskExecutor {
if (!sourceAttachments || sourceAttachments.length === 0) {
return []
}
+ // Frozen provenance identity (#650). `messageId` is the delivery identity
+ // the channel supplied; internal/cron sources carry an empty id, so the
+ // queued task id stands in as the real identity of that task's message.
+ // Neither value is a placeholder, and neither is rewritten here: a value
+ // the shared contract does not accept fails its projection check instead.
+ const messageId = this.task.sourceMessage?.messageId?.trim() || this.task.id
const imageParts = sourceAttachments
.filter(
att =>
@@ -2126,12 +2154,15 @@ export class TaskExecutor {
type: 'image',
mimeType: att.mimeType as 'image/jpeg' | 'image/png',
data: att.dataBase64,
+ source: { kind: 'attachment', attachmentId: att.id, messageId },
})
)
if (!imageParts.length) {
return []
}
- const userText = this.task.sourceMessage?.content?.trim() || 'User attached image(s).'
+ // Preserve the actual history text (including any contextual enrichment).
+ // Reading sourceMessage again would overwrite text already assembled above.
+ const userText = messageContent.trim() ? messageContent : 'User attached image(s).'
return [{ type: 'text', text: userText }, ...imageParts]
}
}
diff --git a/mcp-host/src/config.ts b/mcp-host/src/config.ts
index acfb57cea..0ee7afafe 100644
--- a/mcp-host/src/config.ts
+++ b/mcp-host/src/config.ts
@@ -67,6 +67,8 @@ export interface Config {
// Default OFF. Codex provider construction stays dark until the operator
// sets MCP_HOST_CODEX_SUBSCRIPTION_ENABLED=true.
codexSubscriptionEnabled: boolean
+ /** Models explicitly cleared for visual rollout; empty until interoperability is verified. */
+ codexImageInputModels: string[]
// Server-owned Codex proxy URL. Callers cannot override this per request.
codexProxyRuntimeBaseUrl: string
// Explicit authorize override for tests/dev. Host chat ignores an empty hash
@@ -676,6 +678,10 @@ export const config: Config = {
devModelProvider: devMode ? resolveDevModelProvider(rawDevModelProvider) : undefined,
devModelName,
codexSubscriptionEnabled: process.env.MCP_HOST_CODEX_SUBSCRIPTION_ENABLED === 'true',
+ codexImageInputModels: (process.env.CODEX_IMAGE_INPUT_MODELS ?? '')
+ .split(',')
+ .map(model => model.trim())
+ .filter(Boolean),
codexProxyRuntimeBaseUrl: getEnv(
'CODEX_LLM_PROXY_RUNTIME_URL',
'http://codex-llm-proxy.control-plane.svc.cluster.local:8080'
diff --git a/mcp-host/src/core/adapters/__tests__/llmImageCompatibility.test.ts b/mcp-host/src/core/adapters/__tests__/llmImageCompatibility.test.ts
new file mode 100644
index 000000000..c57a30f41
--- /dev/null
+++ b/mcp-host/src/core/adapters/__tests__/llmImageCompatibility.test.ts
@@ -0,0 +1,735 @@
+/**
+ * #650 — image source identity compatibility across the LlmPort boundary.
+ *
+ * `appendToolResults` keeps one visual part per producing tool call (each with
+ * its own `source`) and marks the parts that only exist to retain a source
+ * identity with `sourceIdentityOnly: true`. The marker is an internal loop
+ * detail: `LlmPortAdapter` must translate it per provider, so a transport that
+ * binds each image source sees every identity while a legacy field-by-field
+ * transport keeps the historical content-deduplicated view.
+ *
+ * These tests drive the REAL `appendToolResults` and the REAL `LlmPortAdapter`
+ * with injected provider stand-ins (no network, no SDK client).
+ */
+import { describe, expect, it, vi } from 'vitest'
+import type {
+ CodexCompletionRequest,
+ CodexMessagePartImageV2,
+} from '@clerum/llm-provider-attempt-contract'
+import type { SingleTurnProvider } from '../../../llm'
+import { JPEG_2X2_BASE64, PNG_2X2_BASE64 } from '../../../llm/__tests__/codexImageFixtures'
+import {
+ type CodexSubscriptionDeps,
+ CodexSubscriptionProvider,
+} from '../../../llm/codexSubscription'
+import { FailoverEngine } from '../../../llm/failover/engine'
+import type { LlmPolicy } from '../../../llm/failover/types'
+import { CodexAuthorizeError } from '../../../llm/providerAttemptAuthorizer'
+import { LlmErrorCode } from '../../errors'
+import { stripHistoricalMedia } from '../../extensions/prePrune'
+import { appendToolResults } from '../../orchestration/toolUseLoopMessages'
+import type { SystemPromptParts } from '../../reasoning/systemPrompt'
+import {
+ type Attachment,
+ type ChatMessage,
+ type CompletionRequest,
+ FinishReason,
+ type MessageContentPart,
+ type ToolDefinition,
+ type ToolResult,
+ prependTextToParts,
+} from '../../types'
+import { maybeWrapFailover } from '../failoverLlmPort'
+import { LlmPortAdapter } from '../llmPortAdapter'
+
+const SYSTEM_PARTS: SystemPromptParts = {
+ stable: 'stable tier',
+ context: 'context tier',
+ stableHash: 'stable-tier-hash',
+ contextHash: 'context-tier-hash',
+}
+
+const TOOLS: ToolDefinition[] = [
+ { name: 'desktop_screenshot', description: 'Capture the desktop', parameters: {} },
+]
+
+const EXPLANATORY_TEXT = 'Here are the screenshots from the tool results above.'
+
+const imageAttachment = (
+ id: string,
+ data = PNG_2X2_BASE64,
+ mimeType: 'image/png' | 'image/jpeg' = 'image/png'
+): Attachment => ({
+ id,
+ kind: 'image',
+ mimeType,
+ encoding: 'base64',
+ dataBase64: data,
+})
+
+const frameResult = (toolCallId: string, attachment: Attachment): ToolResult => ({
+ tool_call_id: toolCallId,
+ name: 'desktop_screenshot',
+ content: `${toolCallId} frame`,
+ is_error: false,
+ attachments: [attachment],
+})
+
+const screenshotResult = (toolCallId: string, attachmentId: string): ToolResult =>
+ frameResult(toolCallId, imageAttachment(attachmentId))
+
+/**
+ * Two tool-loop iterations whose screenshots carry identical bytes, built the
+ * way a source-binding chain does it: the loop forwards
+ * `LoopConfig.imageSourceIdentity` and the caller states it explicitly.
+ */
+function repeatedFrameMessages(preserveSourceIdentity = true): {
+ messages: ChatMessage[]
+ collected: Attachment[]
+} {
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ appendToolResults(
+ messages,
+ [screenshotResult('tc_first', 'att-first')],
+ collected,
+ preserveSourceIdentity
+ )
+ appendToolResults(
+ messages,
+ [screenshotResult('tc_second', 'att-second')],
+ collected,
+ preserveSourceIdentity
+ )
+ return { messages, collected }
+}
+
+type ImagePart = Extract<MessageContentPart, { type: 'image' }>
+
+function imagePartsOf(messages: ChatMessage[]): ImagePart[] {
+ return messages
+ .flatMap(message => message.contentParts ?? [])
+ .filter((part): part is ImagePart => part.type === 'image')
+}
+
+function imageMessagesOf(messages: ChatMessage[]): ChatMessage[] {
+ return messages.filter(
+ message =>
+ message.role === 'user' && (message.contentParts ?? []).some(part => part.type === 'image')
+ )
+}
+
+function explanatoryTextCount(messages: ChatMessage[]): number {
+ return messages
+ .flatMap(message => message.contentParts ?? [])
+ .filter(part => part.type === 'text' && part.text === EXPLANATORY_TEXT).length
+}
+
+type ProviderCall = { parts?: SystemPromptParts; messages: ChatMessage[] }
+
+/**
+ * Stand-in for a `SingleTurnProvider`. `cache: true` adds the optional
+ * cache-aware methods so the adapter routes to them; `imageSourceIdentity`
+ * models a transport that binds every image source (Codex V2).
+ */
+function recordingProvider(options: { cache: boolean; imageSourceIdentity?: boolean }) {
+ const complete: ProviderCall[] = []
+ const completeWithTools: ProviderCall[] = []
+ const base = {
+ getProviderType: () => 'openai' as const,
+ classifyError: () => ({
+ code: LlmErrorCode.ApiCallFailed,
+ retryable: true,
+ message: 'mock',
+ }),
+ ...(options.imageSourceIdentity ? { requiresImageSourceIdentity: true as const } : {}),
+ }
+
+ const stopResponse = () => ({
+ content: 'ok',
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: FinishReason.Stop,
+ })
+ const toolResponse = () => ({
+ content: null,
+ tool_calls: null,
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ finish_reason: FinishReason.Stop,
+ })
+
+ const provider: SingleTurnProvider = options.cache
+ ? {
+ ...base,
+ completeSingleTurn: async messages => {
+ complete.push({ messages })
+ return stopResponse()
+ },
+ completeSingleTurnWithTools: async messages => {
+ completeWithTools.push({ messages })
+ return toolResponse()
+ },
+ completeSingleTurnAndCache: async (parts, messages) => {
+ complete.push({ parts, messages })
+ return stopResponse()
+ },
+ completeSingleTurnWithToolsAndCache: async (parts, messages) => {
+ completeWithTools.push({ parts, messages })
+ return toolResponse()
+ },
+ }
+ : {
+ ...base,
+ completeSingleTurn: async messages => {
+ complete.push({ messages })
+ return stopResponse()
+ },
+ completeSingleTurnWithTools: async messages => {
+ completeWithTools.push({ messages })
+ return toolResponse()
+ },
+ }
+
+ return { provider, complete, completeWithTools }
+}
+
+type DispatchVariant = { label: string; tools: boolean; cache: boolean }
+
+const DISPATCH_VARIANTS: DispatchVariant[] = [
+ { label: 'complete via the cache-aware path', tools: false, cache: true },
+ { label: 'complete via the concat fallback', tools: false, cache: false },
+ { label: 'completeWithTools via the cache-aware path', tools: true, cache: true },
+ { label: 'completeWithTools via the concat fallback', tools: true, cache: false },
+]
+
+async function runVariant(
+ variant: DispatchVariant,
+ imageSourceIdentity: boolean,
+ messages: ChatMessage[]
+): Promise<ProviderCall> {
+ const { provider, complete, completeWithTools } = recordingProvider({
+ cache: variant.cache,
+ imageSourceIdentity,
+ })
+ const adapter = new LlmPortAdapter(provider, 'gpt-4o', 'openai')
+
+ if (variant.tools) {
+ await adapter.completeWithTools({ messages, tools: TOOLS, systemPromptParts: SYSTEM_PARTS })
+ expect(complete).toHaveLength(0)
+ expect(completeWithTools).toHaveLength(1)
+ return completeWithTools[0]
+ }
+
+ await adapter.complete({ messages, systemPromptParts: SYSTEM_PARTS })
+ expect(completeWithTools).toHaveLength(0)
+ expect(complete).toHaveLength(1)
+ return complete[0]
+}
+
+/** The message view the provider actually received, past the concat system part. */
+function providerView(call: ProviderCall, cache: boolean): ChatMessage[] {
+ if (cache) return call.messages
+ expect(call.messages[0]?.role).toBe('system')
+ return call.messages.slice(1)
+}
+
+describe('image source identity compatibility (#650)', () => {
+ it('marks the frames the legacy collection deduplicates while the loop keeps every source', () => {
+ const { messages, collected } = repeatedFrameMessages()
+
+ // The user-facing attachment list keeps its cross-iteration dedup contract.
+ expect(collected).toHaveLength(1)
+
+ const images = imagePartsOf(messages)
+ expect(images.map(image => image.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-first', toolCallId: 'tc_first' },
+ { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ ])
+ expect(images[0].sourceIdentityOnly).toBeUndefined()
+ expect(images[1].sourceIdentityOnly).toBe(true)
+
+ const visualMessages = imageMessagesOf(messages)
+ expect(visualMessages).toHaveLength(2)
+ // The first frame was legacy-retained, so neither its image nor its
+ // explanatory text carries the marker.
+ expect(visualMessages[0].contentParts?.some(part => part.sourceIdentityOnly)).toBe(false)
+ // The second message exists only for the repeated identity: image AND text.
+ expect(visualMessages[1].contentParts).toEqual([
+ { type: 'text', text: EXPLANATORY_TEXT, sourceIdentityOnly: true },
+ expect.objectContaining({ type: 'image', sourceIdentityOnly: true }),
+ ])
+ })
+
+ it.each(DISPATCH_VARIANTS)(
+ 'drops identity-only parts for a provider without the trait on $label',
+ async variant => {
+ const { messages } = repeatedFrameMessages()
+ const call = await runVariant(variant, false, messages)
+ const view = providerView(call, variant.cache)
+
+ // Historical view: exactly the frame the legacy collection retained.
+ expect(imagePartsOf(view)).toEqual([
+ expect.objectContaining({
+ source: { kind: 'tool', attachmentId: 'att-first', toolCallId: 'tc_first' },
+ }),
+ ])
+ expect(imagePartsOf(view)[0].sourceIdentityOnly).toBeUndefined()
+ expect(imageMessagesOf(view)).toHaveLength(1)
+ expect(explanatoryTextCount(view)).toBe(1)
+
+ // The second tool call itself survives; only its extra image message goes.
+ expect(
+ view.filter(message => message.role === 'tool').map(message => message.tool_call_id)
+ ).toEqual(['tc_first', 'tc_second'])
+ }
+ )
+
+ it.each(DISPATCH_VARIANTS)(
+ 'delivers every source identity to a provider with the trait on $label',
+ async variant => {
+ const { messages } = repeatedFrameMessages()
+ const call = await runVariant(variant, true, messages)
+ const view = providerView(call, variant.cache)
+
+ expect(imagePartsOf(view)).toEqual(imagePartsOf(messages))
+ expect(imageMessagesOf(view)).toHaveLength(2)
+ expect(explanatoryTextCount(view)).toBe(2)
+ }
+ )
+
+ it('keeps the retained frame and its text when only one part of a message is identity-only', async () => {
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ // Same bytes, two attachment identities inside ONE tool-result batch.
+ // A source-binding chain states the flag explicitly.
+ appendToolResults(
+ messages,
+ [screenshotResult('tc_one', 'att-a'), screenshotResult('tc_two', 'att-b')],
+ collected,
+ true
+ )
+
+ expect(collected.map(attachment => attachment.id)).toEqual(['att-a'])
+ const visual = imageMessagesOf(messages)
+ expect(visual).toHaveLength(1)
+ expect(visual[0].content).toBe(EXPLANATORY_TEXT)
+ expect(visual[0].contentParts?.map(part => part.sourceIdentityOnly)).toEqual([
+ undefined,
+ undefined,
+ true,
+ ])
+ expect(visual[0].contentParts?.[1]).not.toHaveProperty('sourceIdentityOnly')
+
+ const { provider, complete } = recordingProvider({ cache: true })
+ await new LlmPortAdapter(provider, 'gpt-4o', 'openai').complete({
+ messages,
+ systemPromptParts: SYSTEM_PARTS,
+ })
+ const [filtered] = imageMessagesOf(providerView(complete[0], true))
+
+ // Legacy keeps the first identity and the text; the repeated one is dropped.
+ expect(filtered.content).toBe(EXPLANATORY_TEXT)
+ expect(filtered.contentParts).toEqual([
+ { type: 'text', text: EXPLANATORY_TEXT },
+ expect.objectContaining({
+ type: 'image',
+ source: { kind: 'tool', attachmentId: 'att-a', toolCallId: 'tc_one' },
+ }),
+ ])
+ // The text is present exactly once (no re-derivation of `content` into parts).
+ expect(
+ filtered.contentParts?.filter(part => part.type === 'text').map(part => part.text)
+ ).toEqual([filtered.content])
+
+ const binding = recordingProvider({ cache: true, imageSourceIdentity: true })
+ await new LlmPortAdapter(binding.provider, 'codex-subscription', 'codex-subscription').complete(
+ {
+ messages,
+ systemPromptParts: SYSTEM_PARTS,
+ }
+ )
+ expect(imagePartsOf(binding.complete[0].messages).map(image => image.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-a', toolCallId: 'tc_one' },
+ { kind: 'tool', attachmentId: 'att-b', toolCallId: 'tc_two' },
+ ])
+ })
+
+ it('leaves the canonical request untouched and lets a later adapter see the full view', async () => {
+ const { messages } = repeatedFrameMessages()
+ const snapshot = structuredClone(messages)
+ const request: CompletionRequest = { messages, systemPromptParts: SYSTEM_PARTS }
+
+ const legacy = recordingProvider({ cache: true })
+ await new LlmPortAdapter(legacy.provider, 'gpt-4o', 'openai').complete(request)
+
+ // The filtering produced a provider-specific view, not a mutation.
+ expect(legacy.complete[0].messages).not.toBe(messages)
+ expect(messages).toEqual(snapshot)
+ expect(imagePartsOf(legacy.complete[0].messages)).toHaveLength(1)
+
+ const binding = recordingProvider({ cache: true, imageSourceIdentity: true })
+ await new LlmPortAdapter(binding.provider, 'codex-subscription', 'codex-subscription').complete(
+ request
+ )
+ expect(imagePartsOf(binding.complete[0].messages)).toEqual(imagePartsOf(snapshot))
+ })
+
+ it('applies the same view when the request carries no system prompt parts', async () => {
+ const { messages } = repeatedFrameMessages()
+
+ const legacy = recordingProvider({ cache: false })
+ await new LlmPortAdapter(legacy.provider, 'gpt-4o', 'openai').complete({ messages })
+ expect(imagePartsOf(legacy.complete[0].messages)).toHaveLength(1)
+
+ const legacyTools = recordingProvider({ cache: false })
+ await new LlmPortAdapter(legacyTools.provider, 'gpt-4o', 'openai').completeWithTools({
+ messages,
+ tools: TOOLS,
+ })
+ expect(imagePartsOf(legacyTools.completeWithTools[0].messages)).toHaveLength(1)
+ })
+
+ it('keeps the pure API-key loop on the legacy shape with no markers and no extra message', async () => {
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ // No fourth argument: the loop path for a chain with no source-binding member.
+ appendToolResults(messages, [screenshotResult('tc_first', 'att-first')], collected)
+ appendToolResults(messages, [screenshotResult('tc_second', 'att-second')], collected)
+
+ // The repeated frame adds no message at all, so compaction sees the old shape.
+ expect(collected).toHaveLength(1)
+ expect(messages.map(message => message.role)).toEqual(['tool', 'user', 'tool'])
+ expect(imagePartsOf(messages)).toHaveLength(1)
+ expect(imagePartsOf(messages)[0].source).toEqual({
+ kind: 'tool',
+ attachmentId: 'att-first',
+ toolCallId: 'tc_first',
+ })
+ expect(JSON.stringify(messages)).not.toContain('sourceIdentityOnly')
+
+ const { provider, complete } = recordingProvider({ cache: true })
+ await new LlmPortAdapter(provider, 'gpt-4o', 'openai').complete({ messages })
+
+ // Nothing to translate, so the adapter hands the canonical view straight on.
+ expect(complete[0].messages).toEqual(messages)
+ expect(JSON.stringify(complete[0].messages)).not.toContain('sourceIdentityOnly')
+ })
+
+ it.each([false, true])(
+ 'retains both frames when identical bytes and id carry different filenames (preserveSourceIdentity=%s)',
+ preserveSourceIdentity => {
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_named',
+ name: 'desktop_screenshot',
+ content: 'two names',
+ is_error: false,
+ attachments: [
+ { ...imageAttachment('att-same'), filename: 'first.png' },
+ { ...imageAttachment('att-same'), filename: 'second.png' },
+ ],
+ },
+ ],
+ collected,
+ preserveSourceIdentity
+ )
+
+ // The collection keys on the filename, so neither frame is a duplicate.
+ expect(collected.map(attachment => attachment.filename)).toEqual(['first.png', 'second.png'])
+ const images = imagePartsOf(messages)
+ expect(images).toHaveLength(2)
+ expect(images.map(image => image.sourceIdentityOnly)).toEqual([undefined, undefined])
+ }
+ )
+
+ it.each([false, true])(
+ 'keeps one frame when the same attachment object repeats inside one result (preserveSourceIdentity=%s)',
+ preserveSourceIdentity => {
+ const shared = imageAttachment('att-shared')
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_dup',
+ name: 'desktop_screenshot',
+ content: 'duplicate frame',
+ is_error: false,
+ attachments: [shared, shared],
+ },
+ ],
+ collected,
+ preserveSourceIdentity
+ )
+
+ expect(collected).toHaveLength(1)
+ const images = imagePartsOf(messages)
+ expect(images).toHaveLength(1)
+ expect(images[0].sourceIdentityOnly).toBeUndefined()
+ expect(images[0].source).toEqual({
+ kind: 'tool',
+ attachmentId: 'att-shared',
+ toolCallId: 'tc_dup',
+ })
+ }
+ )
+})
+
+describe('mixed-chain pruning recovery (#650)', () => {
+ it('promotes the surviving frame when pruning removed the legacy representative', async () => {
+ const { messages } = repeatedFrameMessages(true)
+ // protectedTailStart=2 prunes the first frame message and keeps the second.
+ const stripped = stripHistoricalMedia(messages, 2)
+ expect(imagePartsOf(stripped)).toHaveLength(1)
+ expect(imagePartsOf(stripped)[0].source).toEqual({
+ kind: 'tool',
+ attachmentId: 'att-second',
+ toolCallId: 'tc_second',
+ })
+ expect(imagePartsOf(stripped)[0].sourceIdentityOnly).toBe(true)
+ const snapshot = structuredClone(stripped)
+
+ const { provider, complete } = recordingProvider({ cache: true })
+ await new LlmPortAdapter(provider, 'gpt-4o', 'openai').complete({
+ messages: stripped,
+ systemPromptParts: SYSTEM_PARTS,
+ })
+ const view = providerView(complete[0], true)
+
+ // One frame survives without the internal flag, and its text comes back.
+ const visual = imageMessagesOf(view)
+ expect(visual).toHaveLength(1)
+ expect(visual[0].contentParts?.map(part => part.type)).toEqual(['text', 'image'])
+ expect(visual[0].contentParts?.[0]).toEqual({ type: 'text', text: EXPLANATORY_TEXT })
+ expect(imagePartsOf(view)).toHaveLength(1)
+ // The promoted part is the current frame with its internal flag removed.
+ expect(imagePartsOf(view)[0]).toEqual({
+ type: 'image',
+ mimeType: 'image/png',
+ data: PNG_2X2_BASE64,
+ source: { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ })
+ expect(JSON.stringify(view)).not.toContain('sourceIdentityOnly')
+
+ // The adapter cloned: the pruned canonical messages keep their marked frame.
+ expect(stripped).toEqual(snapshot)
+ expect(imagePartsOf(stripped)[0].sourceIdentityOnly).toBe(true)
+ })
+
+ it('promotes one frame per distinct frame when pruning removed both representatives', async () => {
+ const png = imageAttachment('att-x1', PNG_2X2_BASE64)
+ const jpeg = imageAttachment('att-y1', JPEG_2X2_BASE64, 'image/jpeg')
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ appendToolResults(messages, [frameResult('tc_x1', png)], collected, true)
+ appendToolResults(messages, [frameResult('tc_y1', jpeg)], collected, true)
+ // The same bytes from a later call: only the source identity differs.
+ appendToolResults(messages, [frameResult('tc_x2', { ...png, id: 'att-x2' })], collected, true)
+ appendToolResults(messages, [frameResult('tc_y2', { ...jpeg, id: 'att-y2' })], collected, true)
+
+ expect(collected).toHaveLength(2)
+ expect(imagePartsOf(messages).map(image => image.sourceIdentityOnly)).toEqual([
+ undefined,
+ undefined,
+ true,
+ true,
+ ])
+
+ // Both older representatives are pruned; only the marked pair is left.
+ const stripped = stripHistoricalMedia(messages, 4)
+ expect(imagePartsOf(stripped).map(image => image.sourceIdentityOnly)).toEqual([true, true])
+ const snapshot = structuredClone(stripped)
+
+ const { provider, complete } = recordingProvider({ cache: true })
+ await new LlmPortAdapter(provider, 'gpt-4o', 'openai').complete({
+ messages: stripped,
+ systemPromptParts: SYSTEM_PARTS,
+ })
+ const view = providerView(complete[0], true)
+
+ expect(imagePartsOf(view).map(image => image.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-x2', toolCallId: 'tc_x2' },
+ { kind: 'tool', attachmentId: 'att-y2', toolCallId: 'tc_y2' },
+ ])
+ expect(
+ imageMessagesOf(view).map(message => message.contentParts?.map(part => part.type))
+ ).toEqual([
+ ['text', 'image'],
+ ['text', 'image'],
+ ])
+ expect(JSON.stringify(view)).not.toContain('sourceIdentityOnly')
+ expect(stripped).toEqual(snapshot)
+ })
+})
+
+describe('Codex V2 transport wiring (#650)', () => {
+ it('authorizes every source identity and never serializes the internal marker', async () => {
+ const authorize = vi.fn(async (_input: { request: CodexCompletionRequest }) => ({
+ providerAttemptId: 'attempt-1',
+ requestHash: 'a'.repeat(64),
+ executionTicket: 'ticket-123456',
+ expiresAt: '2026-09-17T10:00:00.000Z',
+ }))
+ const stream = vi.fn(
+ async (_input: { request: CodexCompletionRequest; requestHash: string }) => ({
+ text: 'ok',
+ toolCalls: [],
+ outcome: 'success' as const,
+ })
+ )
+ // Infrastructure stand-ins: the authorize gateway and the proxy stream.
+ const provider = new CodexSubscriptionProvider('gpt-5.3-codex', {
+ authorizer: { authorize },
+ proxy: { stream },
+ attemptContext: () => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ }),
+ imageInputEnabled: true,
+ } as unknown as CodexSubscriptionDeps)
+
+ const { messages } = repeatedFrameMessages()
+ // The canonical loop view carries the marker for the repeated identity...
+ expect(imagePartsOf(messages)[1].sourceIdentityOnly).toBe(true)
+ expect(JSON.stringify(messages)).toContain('sourceIdentityOnly')
+
+ await new LlmPortAdapter(provider, 'gpt-5.3-codex', 'codex-subscription').completeWithTools({
+ messages,
+ tools: TOOLS,
+ })
+
+ const wire = authorize.mock.calls[0][0].request
+ if (wire.schemaVersion !== 'codex-completion-request.v2') {
+ throw new Error(`expected a Codex V2 request, received ${wire.schemaVersion}`)
+ }
+ // The class opt-in (not a test-side trait stub) is what keeps both frames:
+ // neither source may be dropped before the request is hashed and authorized.
+ const authorizedImages = wire.messages
+ .flatMap(message => message.contentParts ?? [])
+ .filter((part): part is CodexMessagePartImageV2 => part.type === 'image')
+ expect(authorizedImages.map(image => image.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-first', toolCallId: 'tc_first' },
+ { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ ])
+ // ...while the serialized contract never carries the internal marker.
+ expect(JSON.stringify(wire)).not.toContain('sourceIdentityOnly')
+ // The proxy streams the same authorized request.
+ expect(stream.mock.calls[0][0].request).toBe(wire)
+ })
+
+ it('survives a real cross-provider failover without re-marking or losing a frame', async () => {
+ const authorize = vi.fn(async (_input: { request: CodexCompletionRequest }) => {
+ throw new CodexAuthorizeError('rate_limited', 'gateway answered 429')
+ })
+ const primary = new CodexSubscriptionProvider('gpt-5.3-codex', {
+ authorizer: { authorize },
+ proxy: { stream: vi.fn() },
+ attemptContext: () => ({
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ hostRef: 'chatllm',
+ }),
+ imageInputEnabled: true,
+ } as unknown as CodexSubscriptionDeps)
+ const fallback = recordingProvider({ cache: false })
+ const policy: LlmPolicy = {
+ cooldownSeconds: 300,
+ triggerOn: ['rate_limited'],
+ fallbacks: [{ provider: 'openai', model: 'gpt-5.4' }],
+ }
+ const engine = new FailoverEngine(policy, { metricInc: () => {} })
+
+ const { messages } = repeatedFrameMessages()
+ const snapshot = structuredClone(messages)
+ const wrapped = maybeWrapFailover({
+ primaryPort: new LlmPortAdapter(primary, 'gpt-5.3-codex', 'codex-subscription'),
+ primaryPair: { provider: 'codex-subscription', model: 'gpt-5.3-codex' },
+ engine,
+ policy,
+ buildFallbackPort: () => new LlmPortAdapter(fallback.provider, 'gpt-5.4', 'openai'),
+ })
+
+ await wrapped.completeWithTools({ messages, tools: TOOLS })
+
+ // The failed codex attempt still projected both sources before authorizing.
+ expect(engine.servedBy()).toEqual({ provider: 'openai', model: 'gpt-5.4', fallback: true })
+ const attempted = authorize.mock.calls[0][0].request
+ if (attempted.schemaVersion !== 'codex-completion-request.v2') {
+ throw new Error(`expected a Codex V2 request, received ${attempted.schemaVersion}`)
+ }
+ const attemptedImages = attempted.messages
+ .flatMap(message => message.contentParts ?? [])
+ .filter((part): part is CodexMessagePartImageV2 => part.type === 'image')
+ expect(attemptedImages.map(image => image.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-first', toolCallId: 'tc_first' },
+ { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ ])
+ // The legacy fallback applies its OWN view to the same retried request...
+ expect(imagePartsOf(fallback.completeWithTools[0].messages)).toHaveLength(1)
+ // ...while the canonical messages keep both identities for the next attempt.
+ expect(messages).toEqual(snapshot)
+ expect(imagePartsOf(messages)[1].sourceIdentityOnly).toBe(true)
+ })
+})
+
+describe('identity-only marker preservation', () => {
+ it('keeps the marker when a historical frame is replaced by redaction text', () => {
+ const markedFrame: ChatMessage = {
+ role: 'user',
+ content: EXPLANATORY_TEXT,
+ contentParts: [
+ { type: 'text', text: EXPLANATORY_TEXT, sourceIdentityOnly: true },
+ {
+ type: 'image',
+ mimeType: 'image/png',
+ data: PNG_2X2_BASE64,
+ sourceIdentityOnly: true,
+ source: { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ },
+ ],
+ }
+ const retainedFrame: ChatMessage = {
+ role: 'user',
+ content: EXPLANATORY_TEXT,
+ contentParts: [
+ { type: 'text', text: EXPLANATORY_TEXT },
+ { type: 'image', mimeType: 'image/png', data: PNG_2X2_BASE64 },
+ ],
+ }
+
+ const stripped = stripHistoricalMedia([markedFrame, retainedFrame], 2)
+
+ expect(stripped[0].contentParts?.[1]).toEqual({
+ type: 'text',
+ text: '[image redacted — see turn 0]',
+ sourceIdentityOnly: true,
+ })
+ expect(stripped[1].contentParts?.[1]).toEqual({
+ type: 'text',
+ text: '[image redacted — see turn 1]',
+ })
+ })
+
+ it('keeps the marker when the turn-context prefix rewrites the text part', () => {
+ const parts: MessageContentPart[] = [
+ { type: 'text', text: 'body', sourceIdentityOnly: true },
+ {
+ type: 'image',
+ mimeType: 'image/png',
+ data: PNG_2X2_BASE64,
+ sourceIdentityOnly: true,
+ source: { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ },
+ ]
+
+ expect(prependTextToParts(parts, '<turn-context>\n')).toEqual([
+ { type: 'text', text: '<turn-context>\nbody', sourceIdentityOnly: true },
+ parts[1],
+ ])
+ })
+})
diff --git a/mcp-host/src/core/adapters/__tests__/llmPortAdapterDiagnostics.test.ts b/mcp-host/src/core/adapters/__tests__/llmPortAdapterDiagnostics.test.ts
index 26356b74d..f7bbb0453 100644
--- a/mcp-host/src/core/adapters/__tests__/llmPortAdapterDiagnostics.test.ts
+++ b/mcp-host/src/core/adapters/__tests__/llmPortAdapterDiagnostics.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from 'vitest'
import { SingleTurnProvider } from '../../../llm'
+import { logger } from '../../../logger'
import { LlmError, LlmErrorCode } from '../../errors'
import { LlmPortAdapter } from '../llmPortAdapter'
@@ -7,7 +8,7 @@ describe('LlmPortAdapter diagnostics', () => {
it('does not copy raw cause messages into provider diagnostics logs', async () => {
const cause = new Error('upstream payload included Bearer sk-live-secret')
const originalError = new Error('Connection error.', { cause })
- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const logSpy = vi.spyOn(logger, 'info').mockImplementation(() => undefined)
const mockProvider: SingleTurnProvider = {
completeSingleTurn: vi.fn().mockRejectedValue(originalError),
completeSingleTurnWithTools: vi.fn(),
@@ -22,7 +23,7 @@ describe('LlmPortAdapter diagnostics', () => {
await expect(adapter.complete({ messages: [] })).rejects.toBeInstanceOf(LlmError)
- const logText = logSpy.mock.calls.flat().join('\n')
+ const logText = JSON.stringify(logSpy.mock.calls)
expect(logText).not.toContain('sk-live-secret')
expect(logText).not.toContain('upstream payload included')
logSpy.mockRestore()
@@ -40,7 +41,7 @@ describe('LlmPortAdapter diagnostics', () => {
name: 'APIConnectionError',
code: secretishCode,
})
- const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ const logSpy = vi.spyOn(logger, 'info').mockImplementation(() => undefined)
const mockProvider: SingleTurnProvider = {
completeSingleTurn: vi.fn().mockRejectedValue(originalError),
completeSingleTurnWithTools: vi.fn(),
@@ -55,7 +56,7 @@ describe('LlmPortAdapter diagnostics', () => {
await expect(adapter.complete({ messages: [] })).rejects.toBeInstanceOf(LlmError)
- const logText = logSpy.mock.calls.flat().join('\n')
+ const logText = JSON.stringify(logSpy.mock.calls)
expect(logText).not.toContain(secretishCode)
expect(logText).not.toContain(secretishCauseCode)
expect(logText).toContain(`${secretPrefix}-[redacted]`)
diff --git a/mcp-host/src/core/adapters/llmPortAdapter.ts b/mcp-host/src/core/adapters/llmPortAdapter.ts
index a86f26691..bdf6d745b 100644
--- a/mcp-host/src/core/adapters/llmPortAdapter.ts
+++ b/mcp-host/src/core/adapters/llmPortAdapter.ts
@@ -4,6 +4,7 @@ import {
clerumPromptCacheReadTokens,
clerumPromptCacheWriteTokens,
} from '../../llm/promptCacheMetrics'
+import { logger } from '../../logger'
import { LlmUsageEvent, UsageReporter, newRequestId } from '../../usage/usageReporter.js'
import type { SessionTokenUsage } from '../conversation/conversationStore'
import { LlmError } from '../errors'
@@ -48,7 +49,7 @@ function errorField(value: unknown, field: string): string | null {
return trimmed ? redactDiagnosticField(trimmed).slice(0, 120) : null
}
-function providerErrorDiagnostics(err: unknown): string {
+function providerErrorDiagnostics(err: unknown): Record<string, string> {
const fields: Record<string, string> = {}
const name = errorField(err, 'name')
const type = errorField(err, 'type')
@@ -68,8 +69,7 @@ function providerErrorDiagnostics(err: unknown): string {
if (causeName) fields.causeName = causeName
if (causeCode) fields.causeCode = causeCode
- const entries = Object.entries(fields)
- return entries.length > 0 ? ` diagnostics=${JSON.stringify(Object.fromEntries(entries))}` : ''
+ return fields
}
/**
@@ -122,13 +122,18 @@ export class LlmPortAdapter implements LlmPort {
}
async complete(request: CompletionRequest): Promise<CompletionResponse> {
- console.log(
- `[NewCore:LlmPort] complete → model=${this.model}, messages=${request.messages.length}, cache=${request.systemPromptParts ? 'on' : 'off'}`
+ logger.info(
+ {
+ model: this.model,
+ messageCount: request.messages.length,
+ cache: Boolean(request.systemPromptParts),
+ },
+ 'LLM completion started'
)
const requestId = newRequestId()
try {
const response = await this.dispatchComplete(request)
- console.log(`[NewCore:LlmPort] ← finish_reason=${response.finish_reason}`)
+ logger.info({ finishReason: response.finish_reason }, 'LLM completion finished')
this.recordUsage(requestId, request.usageContext, response.usage)
return response
} catch (err) {
@@ -137,15 +142,22 @@ export class LlmPortAdapter implements LlmPort {
}
async completeWithTools(request: ToolCompletionRequest): Promise<ToolCompletionResponse> {
- console.log(
- `[NewCore:LlmPort] completeWithTools → model=${this.model}, tools=${request.tools.length}, messages=${request.messages.length}, cache=${request.systemPromptParts ? 'on' : 'off'}`
+ logger.info(
+ {
+ model: this.model,
+ toolCount: request.tools.length,
+ messageCount: request.messages.length,
+ cache: Boolean(request.systemPromptParts),
+ },
+ 'LLM tool completion started'
)
const requestId = newRequestId()
try {
const response = await this.dispatchCompleteWithTools(request)
const toolCallCount = response.tool_calls?.length ?? 0
- console.log(
- `[NewCore:LlmPort] ← finish_reason=${response.finish_reason}, tool_calls=${toolCallCount}, usage=${JSON.stringify(response.usage ?? {})}`
+ logger.info(
+ { finishReason: response.finish_reason, toolCallCount, usage: response.usage },
+ 'LLM tool completion finished'
)
this.recordUsage(requestId, request.usageContext, response.usage)
return response
@@ -155,23 +167,65 @@ export class LlmPortAdapter implements LlmPort {
}
/**
- * T2.2 — route to the provider's cache-aware path when both
- * `request.systemPromptParts` is set AND the provider implements the
- * cache-aware method (Claude). Otherwise concat the parts back into a
- * single `system` message and call the legacy method (OpenAI / ZAI /
- * Bailian don't expose explicit cache markers; they cache implicitly by
- * prefix when the routing is stable).
+ * Preserve the historical content-deduplicated API-key view without changing
+ * the loop's canonical messages. Each fallback adapter selects its own view;
+ * source-binding transports retain all identities before hashing them.
*/
+ private providerMessages(messages: ChatMessage[]): ChatMessage[] {
+ if (
+ this.provider.requiresImageSourceIdentity ||
+ !messages.some(message => message.contentParts?.some(part => part.sourceIdentityOnly))
+ ) {
+ return messages
+ }
+ const imageKey = (part: { mimeType: string; data: string }) =>
+ `${part.mimeType}\u0000${part.data}`
+ const represented = new Set<string>()
+ for (const message of messages) {
+ for (const part of message.contentParts ?? []) {
+ if (part.type === 'image' && !part.sourceIdentityOnly) represented.add(imageKey(part))
+ }
+ }
+ return messages.flatMap(message => {
+ if (!message.contentParts?.some(part => part.sourceIdentityOnly)) return [message]
+ let restoredImage = false
+ const selected = new Set(message.contentParts.filter(part => !part.sourceIdentityOnly))
+ for (const part of message.contentParts) {
+ if (part.type !== 'image' || !part.sourceIdentityOnly || represented.has(imageKey(part)))
+ continue
+ // In a mixed chain, pruning can remove the older legacy representative.
+ // Retain one current frame rather than silently losing all visual input.
+ selected.add(part)
+ represented.add(imageKey(part))
+ restoredImage = true
+ }
+ if (restoredImage) {
+ for (const part of message.contentParts) if (part.type === 'text') selected.add(part)
+ }
+ const contentParts = message.contentParts
+ .filter(part => selected.has(part))
+ .map(part => {
+ if (!part.sourceIdentityOnly) return part
+ const visible = { ...part }
+ delete visible.sourceIdentityOnly
+ return visible
+ })
+ return contentParts.length ? [{ ...message, contentParts }] : []
+ })
+ }
+
+ /** Native cache markers when supported, otherwise the existing system-text projection. */
private async dispatchComplete(request: CompletionRequest): Promise<CompletionResponse> {
const parts = request.systemPromptParts
+ const providerMessages = this.providerMessages(request.messages)
if (parts && this.provider.completeSingleTurnAndCache) {
- return this.provider.completeSingleTurnAndCache(parts, request.messages, {
+ return this.provider.completeSingleTurnAndCache(parts, providerMessages, {
max_tokens: request.max_tokens,
temperature: request.temperature,
signal: request.signal,
})
}
- const messages = parts ? prependConcatSystem(parts, request.messages) : request.messages
+ const messages = parts ? prependConcatSystem(parts, providerMessages) : providerMessages
return this.provider.completeSingleTurn(messages, {
max_tokens: request.max_tokens,
temperature: request.temperature,
@@ -183,10 +237,11 @@ export class LlmPortAdapter implements LlmPort {
request: ToolCompletionRequest
): Promise<ToolCompletionResponse> {
const parts = request.systemPromptParts
+ const providerMessages = this.providerMessages(request.messages)
if (parts && this.provider.completeSingleTurnWithToolsAndCache) {
return this.provider.completeSingleTurnWithToolsAndCache(
parts,
- request.messages,
+ providerMessages,
request.tools,
{
max_tokens: request.max_tokens,
@@ -196,7 +251,7 @@ export class LlmPortAdapter implements LlmPort {
}
)
}
- const messages = parts ? prependConcatSystem(parts, request.messages) : request.messages
+ const messages = parts ? prependConcatSystem(parts, providerMessages) : providerMessages
return this.provider.completeSingleTurnWithTools(messages, request.tools, {
max_tokens: request.max_tokens,
temperature: request.temperature,
@@ -251,7 +306,10 @@ export class LlmPortAdapter implements LlmPort {
cache_write_tokens: usage.cache_write_tokens,
})
} catch (err) {
- console.error('[NewCore:LlmPort] onUsageRecorded sink threw (ignored):', err)
+ logger.error(
+ { err: providerErrorDiagnostics(err) },
+ 'Usage sink failed; completion preserved'
+ )
}
}
if (!this.usageReporter || !this.staticContext || !usageContext || !usage) return
@@ -292,10 +350,13 @@ export class LlmPortAdapter implements LlmPort {
*/
private handleProviderError(err: unknown): never {
const classified = this.provider.classifyError(err)
- console.log(
- `[NewCore:LlmPort] ← ERROR: code=${classified.code} ` +
- `retryable=${classified.retryable} message="${classified.message}"` +
- providerErrorDiagnostics(err)
+ logger.info(
+ {
+ code: classified.code,
+ retryable: classified.retryable,
+ diagnostics: providerErrorDiagnostics(err),
+ },
+ 'LLM provider call failed'
)
throw new LlmError(
classified.message,
diff --git a/mcp-host/src/core/extensions/prePrune.ts b/mcp-host/src/core/extensions/prePrune.ts
index 6c3ce9c2f..c65acee45 100644
--- a/mcp-host/src/core/extensions/prePrune.ts
+++ b/mcp-host/src/core/extensions/prePrune.ts
@@ -399,7 +399,13 @@ export function stripHistoricalMedia(
const hasImage = msg.contentParts.some(p => p.type === 'image')
if (!hasImage) continue
const newParts: MessageContentPart[] = msg.contentParts.map(part =>
- part.type === 'image' ? { type: 'text', text: `[image redacted — see turn ${i}]` } : part
+ part.type === 'image'
+ ? {
+ type: 'text',
+ text: `[image redacted — see turn ${i}]`,
+ ...(part.sourceIdentityOnly ? { sourceIdentityOnly: true as const } : {}),
+ }
+ : part
)
if (mutated === null) mutated = messages.slice()
mutated[i] = { ...msg, contentParts: newParts }
diff --git a/mcp-host/src/core/orchestration/__tests__/toolUseLoopMessages.test.ts b/mcp-host/src/core/orchestration/__tests__/toolUseLoopMessages.test.ts
index 7423e3100..4de21e464 100644
--- a/mcp-host/src/core/orchestration/__tests__/toolUseLoopMessages.test.ts
+++ b/mcp-host/src/core/orchestration/__tests__/toolUseLoopMessages.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
+import { PNG_2X2_BASE64 } from '../../../llm/__tests__/codexImageFixtures'
import type { Attachment, ChatMessage, ToolResult } from '../../types'
import { appendToolResults } from '../toolUseLoopMessages'
@@ -218,4 +219,151 @@ describe('appendToolResults', () => {
expect(collectedAttachments).toEqual([])
})
+
+ const imageAttachment = (id: string, data = PNG_2X2_BASE64): Attachment => ({
+ id,
+ kind: 'image',
+ mimeType: 'image/png',
+ encoding: 'base64',
+ dataBase64: data,
+ })
+
+ it('carries the producing tool call as provenance and keeps content equal to the text parts', () => {
+ const messages: ChatMessage[] = []
+ const collectedAttachments: Attachment[] = []
+
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_frame',
+ name: 'desktop_screenshot',
+ content: 'screenshot taken',
+ is_error: false,
+ attachments: [imageAttachment('att-frame')],
+ },
+ ],
+ collectedAttachments,
+ true
+ )
+
+ expect(messages[0]).toEqual({
+ role: 'tool',
+ content: 'screenshot taken',
+ tool_call_id: 'tc_frame',
+ name: 'desktop_screenshot',
+ })
+ const visual = messages[1]
+ expect(visual.role).toBe('user')
+ expect(visual.content).toBe('Here are the screenshots from the tool results above.')
+ expect(visual.contentParts).toEqual([
+ { type: 'text', text: visual.content },
+ {
+ type: 'image',
+ mimeType: 'image/png',
+ data: PNG_2X2_BASE64,
+ source: { kind: 'tool', attachmentId: 'att-frame', toolCallId: 'tc_frame' },
+ },
+ ])
+ })
+
+ it('keeps the same bytes from two distinct tool calls as two visual parts', () => {
+ const messages: ChatMessage[] = []
+ const collectedAttachments: Attachment[] = []
+
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_first',
+ name: 'desktop_screenshot',
+ content: 'first',
+ is_error: false,
+ attachments: [imageAttachment('att-first')],
+ },
+ ],
+ collectedAttachments,
+ true
+ )
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_second',
+ name: 'desktop_screenshot',
+ content: 'second',
+ is_error: false,
+ attachments: [imageAttachment('att-second')],
+ },
+ ],
+ collectedAttachments,
+ true
+ )
+
+ // The user-facing list still collapses the repeated frame...
+ expect(collectedAttachments).toHaveLength(1)
+ // ...while the model sees one frame per tool call that produced it.
+ const visualMessages = messages.filter(message => message.contentParts !== undefined)
+ expect(visualMessages).toHaveLength(2)
+ expect(visualMessages.map(message => message.contentParts![1])).toEqual([
+ expect.objectContaining({
+ source: { kind: 'tool', attachmentId: 'att-first', toolCallId: 'tc_first' },
+ }),
+ expect.objectContaining({
+ source: { kind: 'tool', attachmentId: 'att-second', toolCallId: 'tc_second' },
+ }),
+ ])
+ })
+
+ it('does not repeat the same frame twice for one tool call', () => {
+ const messages: ChatMessage[] = []
+ const collectedAttachments: Attachment[] = []
+
+ appendToolResults(
+ messages,
+ [
+ {
+ tool_call_id: 'tc_repeat',
+ name: 'desktop_screenshot',
+ content: 'twice',
+ is_error: false,
+ attachments: [imageAttachment('att-a'), imageAttachment('att-a')],
+ },
+ ],
+ collectedAttachments,
+ true
+ )
+
+ expect(collectedAttachments).toHaveLength(1)
+ expect(messages).toHaveLength(2)
+ expect(messages[1].contentParts).toHaveLength(2)
+ expect(messages[1].contentParts![1]).toEqual(
+ expect.objectContaining({
+ source: { kind: 'tool', attachmentId: 'att-a', toolCallId: 'tc_repeat' },
+ })
+ )
+ })
+
+ it('preserves separate attachment identities in one call and avoids visual re-delivery', () => {
+ const messages: ChatMessage[] = []
+ const collected: Attachment[] = []
+ const result = {
+ tool_call_id: 'tc_identity',
+ name: 'desktop_screenshot',
+ content: 'frames',
+ is_error: false,
+ attachments: [imageAttachment('att-a'), imageAttachment('att-b')],
+ }
+ appendToolResults(messages, [result], collected, true)
+ appendToolResults(messages, [result], collected, true)
+ expect(collected).toHaveLength(1)
+ const images = messages
+ .flatMap(message => message.contentParts ?? [])
+ .filter(part => part.type === 'image')
+ expect(images).toHaveLength(2)
+ expect(images.map(part => part.source)).toEqual([
+ { kind: 'tool', attachmentId: 'att-a', toolCallId: 'tc_identity' },
+ { kind: 'tool', attachmentId: 'att-b', toolCallId: 'tc_identity' },
+ ])
+ })
})
diff --git a/mcp-host/src/core/orchestration/loopConfig.ts b/mcp-host/src/core/orchestration/loopConfig.ts
index f938111a0..14cecda49 100644
--- a/mcp-host/src/core/orchestration/loopConfig.ts
+++ b/mcp-host/src/core/orchestration/loopConfig.ts
@@ -21,6 +21,8 @@ import { Attachment, ChatMessage, Conversation, PendingApproval, ToolDefinition
* All extension hooks have passthrough defaults. Override any subset.
*/
export interface LoopConfig {
+ /** Preserve source identities only for chains whose request contract needs them. */
+ imageSourceIdentity?: boolean
// Core dependencies
reasoning: ReasoningPort
toolRegistry: ToolRegistry
diff --git a/mcp-host/src/core/orchestration/toolUseLoop.ts b/mcp-host/src/core/orchestration/toolUseLoop.ts
index 605802bad..93196a4b1 100644
--- a/mcp-host/src/core/orchestration/toolUseLoop.ts
+++ b/mcp-host/src/core/orchestration/toolUseLoop.ts
@@ -341,7 +341,12 @@ export async function runToolUseLoop(
result.usage
)
if (cancelled) {
- appendToolResults(messages, toolResults, collectedAttachments)
+ appendToolResults(
+ messages,
+ toolResults,
+ collectedAttachments,
+ config.imageSourceIdentity === true
+ )
return { type: 'cancelled', reason: 'signal_aborted' }
}
if (pendingApproval) {
@@ -386,7 +391,12 @@ export async function runToolUseLoop(
)
}
- appendToolResults(messages, toolResults, collectedAttachments)
+ appendToolResults(
+ messages,
+ toolResults,
+ collectedAttachments,
+ config.imageSourceIdentity === true
+ )
lastToolResults = toolResults
messages = await manageMessagesForIteration(config, messages, iteration)
validateToolLinkages(messages)
diff --git a/mcp-host/src/core/orchestration/toolUseLoopMessages.ts b/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
index 4ddf20ece..4518f3d1b 100644
--- a/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
+++ b/mcp-host/src/core/orchestration/toolUseLoopMessages.ts
@@ -54,6 +54,50 @@ export function collectToolAttachments(
return added
}
+/** Text of the user message that carries tool-result frames. */
+const TOOL_RESULT_IMAGE_TEXT = 'Here are the screenshots from the tool results above.'
+
+/**
+ * Wire-eligible images of ONE tool result, with their provenance.
+ *
+ * Deduplication deliberately differs from the UI collection above: the
+ * user-facing attachment list collapses identical bytes across iterations,
+ * while the model must see every distinct tool call that produced a frame. Two
+ * calls returning the same bytes can yield two parts with different sources.
+ * The default loop keeps the historical collection view; source binding is
+ * explicitly enabled by the transport capabilities of the configured chain.
+ */
+function collectVisualImageParts(
+ result: ToolResult,
+ seen: Set<string>,
+ legacyRetained: Set<Attachment>,
+ preserveSourceIdentity: boolean
+): MessageContentPart[] {
+ const parts: MessageContentPart[] = []
+ for (const attachment of result.attachments ?? []) {
+ if (!shouldCollectAttachment(result, attachment)) continue
+ if (attachment.kind !== 'image') continue
+ const mimeType = attachment.mimeType
+ if (mimeType !== 'image/jpeg' && mimeType !== 'image/png') continue
+ // Consume membership once: an array can repeat the same attachment object.
+ const retained = legacyRetained.delete(attachment)
+ if (!retained && !preserveSourceIdentity) continue
+ const key = `${result.tool_call_id}\u0000${attachment.id}\u0000${mimeType}\u0000${attachment.dataBase64}`
+ // Keep every frame the original collection retained (including distinct
+ // filenames/lanes); source dedup must not narrow that legacy view.
+ if (seen.has(key) && !retained) continue
+ seen.add(key)
+ parts.push({
+ type: 'image',
+ mimeType,
+ data: attachment.dataBase64,
+ source: { kind: 'tool', attachmentId: attachment.id, toolCallId: result.tool_call_id },
+ ...(!retained ? { sourceIdentityOnly: true as const } : {}),
+ })
+ }
+ return parts
+}
+
export function mergeCollectedAttachments(
collected: Attachment[],
attachments: Attachment[]
@@ -64,22 +108,29 @@ export function mergeCollectedAttachments(
export function appendToolResults(
messages: ChatMessage[],
toolResults: ToolResult[],
- collectedAttachments: Attachment[]
+ collectedAttachments: Attachment[],
+ preserveSourceIdentity = false
): void {
const pendingImages: MessageContentPart[] = []
- for (const tr of toolResults) {
- const trustedAttachments = collectToolAttachments([tr], collectedAttachments)
- if (trustedAttachments.length) {
- for (const att of trustedAttachments) {
- if (att.kind !== 'image') continue
- if (att.mimeType !== 'image/jpeg' && att.mimeType !== 'image/png') continue
- pendingImages.push({
- type: 'image',
- mimeType: att.mimeType,
- data: att.dataBase64,
- })
+ const seenVisuals = new Set<string>()
+ if (preserveSourceIdentity) {
+ for (const message of messages) {
+ for (const part of message.contentParts ?? []) {
+ if (part.type !== 'image' || part.source?.kind !== 'tool') continue
+ seenVisuals.add(
+ `${part.source.toolCallId}\u0000${part.source.attachmentId}\u0000${part.mimeType}\u0000${part.data}`
+ )
}
}
+ }
+ for (const tr of toolResults) {
+ // The UI collection keeps its cross-iteration dedup contract; the visual
+ // parts are collected independently so a repeated frame still carries the
+ // tool call that produced THIS instance.
+ const legacyRetained = new Set(collectToolAttachments([tr], collectedAttachments))
+ pendingImages.push(
+ ...collectVisualImageParts(tr, seenVisuals, legacyRetained, preserveSourceIdentity)
+ )
messages.push({
role: 'tool',
content: tr.content,
@@ -94,9 +145,15 @@ export function appendToolResults(
if (pendingImages.length > 0) {
messages.push({
role: 'user',
- content: 'Here are the screenshots from the tool results above.',
+ content: TOOL_RESULT_IMAGE_TEXT,
contentParts: [
- { type: 'text', text: 'Here are the screenshots from the tool results above.' },
+ {
+ type: 'text',
+ text: TOOL_RESULT_IMAGE_TEXT,
+ ...(pendingImages.every(part => part.sourceIdentityOnly)
+ ? { sourceIdentityOnly: true as const }
+ : {}),
+ },
...pendingImages,
],
})
diff --git a/mcp-host/src/core/types.ts b/mcp-host/src/core/types.ts
index 25fd01aac..7d135ffa2 100644
--- a/mcp-host/src/core/types.ts
+++ b/mcp-host/src/core/types.ts
@@ -12,14 +12,43 @@ import type { SystemPromptParts } from './reasoning/systemPrompt'
export type MessageRole = 'system' | 'user' | 'assistant' | 'tool'
+/**
+ * Wire provenance of an image part (issue #650). The Host resolves it from the
+ * trusted attachment that produced the bytes, and the Codex V2 contract carries
+ * it so a visual part stays attributable to the message or tool call it came
+ * from. Provenance never reaches a provider that maps parts field by field.
+ */
+export type MessageContentImageSource =
+ | { kind: 'attachment'; attachmentId: string; messageId: string }
+ | { kind: 'tool'; attachmentId: string; toolCallId: string }
+
+/** sourceIdentityOnly is internal projection metadata; it never belongs to the provider wire. */
export type MessageContentPart =
- | { type: 'text'; text: string }
- | { type: 'image'; mimeType: 'image/jpeg' | 'image/png'; data: string }
+ | { type: 'text'; text: string; sourceIdentityOnly?: true }
+ | {
+ type: 'image'
+ mimeType: 'image/jpeg' | 'image/png'
+ data: string
+ /** Extra source copy, omitted from the legacy view while another current frame represents it. */
+ sourceIdentityOnly?: true
+ /**
+ * Optional so pre-#650 producers and provider translators keep compiling;
+ * `codexSubscription` rejects an image part without a usable source
+ * instead of shipping an unattributable frame.
+ */
+ source?: MessageContentImageSource
+ }
export interface ChatMessage {
role: MessageRole
content: string
- contentParts?: MessageContentPart[] // only set when tool results have images
+ /**
+ * Set when the turn carries images (user attachments or tool-result frames).
+ * Invariant: when parts are present, `content` is the '\n'-join of the text
+ * parts. The Codex V2 parser enforces that equality, and the local producers
+ * below keep both fields in sync.
+ */
+ contentParts?: MessageContentPart[]
tool_call_id?: string
name?: string
tool_calls?: ToolCall[] | null
@@ -32,6 +61,35 @@ export interface ChatMessage {
spillover_ref?: string
}
+/**
+ * Authoritative text of a message whose parts are present. Codex V2 defines
+ * `content` as the '\n'-join of the text parts, so producers and the provider
+ * adapter derive it from the same place instead of trusting two copies that can
+ * disagree after redaction.
+ */
+export function textContentFromParts(parts: MessageContentPart[]): string {
+ return parts
+ .filter((part): part is Extract<MessageContentPart, { type: 'text' }> => part.type === 'text')
+ .map(part => part.text)
+ .join('\n')
+}
+
+/**
+ * Prepend `prefix` to the text carried by `parts` without reordering them. The
+ * `<turn-context>` block uses this so a message with images keeps `content` and
+ * its text parts equal after the prepend.
+ */
+export function prependTextToParts(
+ parts: MessageContentPart[],
+ prefix: string
+): MessageContentPart[] {
+ const index = parts.findIndex(part => part.type === 'text')
+ if (index < 0) return [{ type: 'text', text: prefix }, ...parts]
+ return parts.map((part, i) =>
+ i === index && part.type === 'text' ? { ...part, text: prefix + part.text } : part
+ )
+}
+
export interface ToolCall {
id: string
name: string
diff --git a/mcp-host/src/llm/__tests__/codexImageFixtures.ts b/mcp-host/src/llm/__tests__/codexImageFixtures.ts
new file mode 100644
index 000000000..09dd7a1e2
--- /dev/null
+++ b/mcp-host/src/llm/__tests__/codexImageFixtures.ts
@@ -0,0 +1,24 @@
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+// Shared, independently decoded 2x2 fixtures; structural acceptance alone
+// does not prove that a header-only substitute actually decodes.
+const fixtures = JSON.parse(
+ readFileSync(
+ join(
+ __dirname,
+ '../../../../packages/llm-provider-attempt-contract/fixtures/visual-requests.json'
+ ),
+ 'utf8'
+ )
+)
+const imageData = (format: 'png' | 'jpeg'): string => {
+ const part = fixtures[format].messages[0].contentParts.find(
+ (entry: { type: string }) => entry.type === 'image'
+ )
+ if (!part?.data) throw new Error(`Missing ${format} image fixture`)
+ return part.data
+}
+
+export const PNG_2X2_BASE64 = imageData('png')
+export const JPEG_2X2_BASE64 = imageData('jpeg')
diff --git a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
index 583455635..37146aef8 100644
--- a/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
+++ b/mcp-host/src/llm/__tests__/codexLlmProxyClient.test.ts
@@ -1,4 +1,9 @@
import { describe, expect, it, vi } from 'vitest'
+import {
+ buildCodexProxyEnvelope,
+ hashCodexCompletionRequest,
+ parseCodexCompletionRequest,
+} from '@clerum/llm-provider-attempt-contract'
import { CodexLlmProxyClient, resolveCodexProxyRuntimeUrl } from '../codexLlmProxyClient'
function sse(frames: unknown[]): ReadableStream<Uint8Array> {
@@ -13,6 +18,76 @@ function sse(frames: unknown[]): ReadableStream<Uint8Array> {
}
describe('CodexLlmProxyClient', () => {
+ it('sends the exact measured V2 envelope and refuses an independent deadline or hash', async () => {
+ const parsed = parseCodexCompletionRequest({
+ schemaVersion: 'codex-completion-request.v2',
+ requestId: 'req-visual',
+ idempotencyKey: 'idem-visual',
+ provider: 'codex-subscription',
+ model: 'gpt-5.1',
+ deadlineMs: 1000,
+ messages: [
+ {
+ role: 'user',
+ content: 'image redacted',
+ contentParts: [{ type: 'text', text: 'image redacted' }],
+ },
+ ],
+ })
+ if (!parsed.ok) throw new Error(parsed.message)
+ const input = {
+ request: parsed.value,
+ requestHash: hashCodexCompletionRequest(parsed.value),
+ executionTicket: 'fixture-ticket',
+ }
+ const measured = buildCodexProxyEnvelope(input)
+ if (!measured.ok) throw new Error(measured.message)
+ const fetchFn = vi.fn<typeof fetch>(
+ async () => new Response(sse([{ type: 'done', outcome: 'success' }]))
+ )
+ const client = new CodexLlmProxyClient({
+ runtimeUrl: 'http://proxy/completions',
+ readPlatformJwt: () => 'fixture-only',
+ fetchFn,
+ })
+ await client.stream({ ...input, deadlineMs: 1000 })
+ expect(JSON.parse(String(fetchFn.mock.calls[0]?.[1]?.body))).toEqual(measured.value)
+ expect(measured.value).not.toHaveProperty('deadlineMs')
+ await expect(client.stream({ ...input, deadlineMs: 2000 })).rejects.toMatchObject({
+ code: 'invalid_request',
+ dispatched: false,
+ })
+ await expect(client.stream({ ...input, requestHash: 'a'.repeat(64) })).rejects.toMatchObject({
+ code: 'request_hash_mismatch',
+ dispatched: false,
+ })
+ expect(fetchFn).toHaveBeenCalledOnce()
+ })
+
+ it.each(['html', 'json'])(
+ 'keeps a %s 413 terminal rather than treating it as unavailable',
+ async format => {
+ const fetchFn = vi.fn(
+ async () =>
+ new Response(
+ format === 'html'
+ ? '<h1>Too large</h1>'
+ : JSON.stringify({ error: 'payload_too_large' }),
+ { status: 413 }
+ )
+ )
+ const client = new CodexLlmProxyClient({
+ runtimeUrl: 'http://proxy/internal/runtime/v1/codex/completions',
+ readPlatformJwt: () => 'test-jwt',
+ fetchFn,
+ })
+ await expect(
+ client.stream({ executionTicket: 'test-ticket', requestHash: 'a'.repeat(64), request: {} })
+ ).rejects.toMatchObject({ code: 'payload_too_large', dispatched: true })
+ expect(fetchFn).toHaveBeenCalledOnce()
+ }
+ )
+
it('streams to the frozen runtime Service URL and never accepts a caller URL', async () => {
const fetchFn = vi.fn().mockResolvedValue({
ok: true,
diff --git a/mcp-host/src/llm/__tests__/codexSubscription.test.ts b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
index 79cfdab0b..0d2147c6e 100644
--- a/mcp-host/src/llm/__tests__/codexSubscription.test.ts
+++ b/mcp-host/src/llm/__tests__/codexSubscription.test.ts
@@ -1,4 +1,8 @@
import { describe, expect, it, vi } from 'vitest'
+import {
+ hashCodexCompletionRequest,
+ parseCodexCompletionRequest,
+} from '@clerum/llm-provider-attempt-contract'
import {
buildToolDescribeResponse,
buildToolSearchResponse,
@@ -9,11 +13,13 @@ import {
import { LlmErrorCode } from '../../core/errors'
import { DeferrableToolController } from '../../core/orchestration/deferrableToolController'
import { DefaultLoopController } from '../../core/orchestration/loopConfig'
+import type { ChatMessage } from '../../core/types'
import { CodexProxyError } from '../codexLlmProxyClient'
import { CodexSubscriptionProvider } from '../codexSubscription'
import { classifyFailoverClass } from '../failover/classify'
import { CodexAuthorizeError } from '../providerAttemptAuthorizer'
import { makeProvider } from '../registry'
+import { JPEG_2X2_BASE64, PNG_2X2_BASE64 } from './codexImageFixtures'
const requestHash = 'a'.repeat(64)
@@ -435,4 +441,250 @@ describe('CodexSubscriptionProvider', () => {
await provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
expect(wired.authorize).toHaveBeenNthCalledWith(2, expect.any(Object), {})
})
+
+ // ─── #650: image input projection ─────────────────────────────────────────
+
+ const METHODS = ['completeSingleTurn', 'completeSingleTurnWithTools'] as const
+ type Method = (typeof METHODS)[number]
+
+ function invoke(
+ provider: CodexSubscriptionProvider,
+ method: Method,
+ messages: ChatMessage[]
+ ): Promise<unknown> {
+ const tools = [{ name: 'echo', description: 'echo', parameters: { type: 'object' } }]
+ return method === 'completeSingleTurn'
+ ? provider.completeSingleTurn(messages)
+ : provider.completeSingleTurnWithTools(messages, tools)
+ }
+
+ function userWithImage(
+ data = PNG_2X2_BASE64,
+ mimeType: 'image/png' | 'image/jpeg' = 'image/png'
+ ): ChatMessage[] {
+ return [
+ {
+ role: 'user',
+ content: 'what is on screen?',
+ contentParts: [
+ { type: 'text', text: 'what is on screen?' },
+ {
+ type: 'image',
+ mimeType,
+ data,
+ source: { kind: 'attachment', attachmentId: 'att-1', messageId: 'msg-1' },
+ },
+ ],
+ },
+ ]
+ }
+
+ it.each(METHODS)('sends a real PNG as a V2 part through %s', async method => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+
+ await invoke(provider, method, userWithImage())
+
+ const authorized = wired.authorize.mock.calls[0][0]
+ expect(authorized.request.schemaVersion).toBe('codex-completion-request.v2')
+ expect(authorized.request.messages[0]).toEqual({
+ role: 'user',
+ content: 'what is on screen?',
+ contentParts: [
+ { type: 'text', text: 'what is on screen?' },
+ {
+ type: 'image',
+ mimeType: 'image/png',
+ data: PNG_2X2_BASE64,
+ source: { kind: 'attachment', attachmentId: 'att-1', messageId: 'msg-1' },
+ },
+ ],
+ })
+ // The authorized hash and the dispatched request are the same canonical
+ // projection the proxy and control-api re-derive from these bytes.
+ const parsed = parseCodexCompletionRequest(authorized.request)
+ expect(parsed.ok).toBe(true)
+ if (parsed.ok) {
+ expect(authorized.requestHash).toBe(hashCodexCompletionRequest(parsed.value))
+ }
+ expect(wired.stream.mock.calls[0][0].request).toEqual(authorized.request)
+ })
+
+ it('accepts a JPEG part when image input is enabled', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+
+ await provider.completeSingleTurn(userWithImage(JPEG_2X2_BASE64, 'image/jpeg'))
+
+ const parts = wired.authorize.mock.calls[0][0].request.messages[0].contentParts
+ expect(parts[1]).toEqual(
+ expect.objectContaining({ mimeType: 'image/jpeg', data: JPEG_2X2_BASE64 })
+ )
+ })
+
+ it.each(METHODS)(
+ 'refuses to authorize an image through %s while image input is dark',
+ async method => {
+ const wired = deps()
+ // `imageInputEnabled` is absent: the documented default is false.
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', wired as never)
+
+ await expect(invoke(provider, method, userWithImage())).rejects.toMatchObject({
+ name: 'CodexAuthorizeError',
+ code: 'image_input_unsupported',
+ })
+ expect(wired.authorize).not.toHaveBeenCalled()
+ expect(wired.stream).not.toHaveBeenCalled()
+ }
+ )
+
+ it('classifies an unsupported visual request as terminal with no fallback', () => {
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', deps() as never)
+ const classified = provider.classifyError(
+ new CodexAuthorizeError('image_input_unsupported', 'image input is not enabled')
+ )
+ expect(classified.code).toBe(LlmErrorCode.ApiCallFailed)
+ expect(classified.retryable).toBe(false)
+ expect(classified.providerDispatched).toBe(false)
+ expect(classifyFailoverClass(classified.code, classified.retryable)).toBeNull()
+ })
+
+ it.each(METHODS)('rejects an image part without provenance through %s', async method => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+ const messages = userWithImage()
+ messages[0].contentParts = [
+ { type: 'text', text: 'what is on screen?' },
+ { type: 'image', mimeType: 'image/png', data: PNG_2X2_BASE64 },
+ ]
+
+ await expect(invoke(provider, method, messages)).rejects.toMatchObject({
+ name: 'CodexAuthorizeError',
+ code: 'image_source_invalid',
+ })
+ expect(wired.authorize).not.toHaveBeenCalled()
+ expect(wired.stream).not.toHaveBeenCalled()
+ })
+
+ it('rejects parts on a non-user message instead of dropping them', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+
+ await expect(
+ provider.completeSingleTurn([
+ { role: 'user', content: 'hi' },
+ {
+ role: 'assistant',
+ content: 'look',
+ contentParts: [{ type: 'text', text: 'look' }],
+ },
+ ])
+ ).rejects.toMatchObject({ name: 'CodexAuthorizeError', code: 'invalid_request' })
+ expect(wired.authorize).not.toHaveBeenCalled()
+ })
+
+ it('keeps a redacted text-only message on V2 without requiring image input', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', wired as never)
+ const messages = userWithImage()
+ // What pre-prune leaves behind: the image became a text part and `content`
+ // is restated from the parts.
+ messages[0].contentParts = [
+ { type: 'text', text: 'what is on screen?' },
+ { type: 'text', text: '[image redacted — see turn 1]' },
+ ]
+ messages[0].content = 'what is on screen?\n[image redacted — see turn 1]'
+
+ await provider.completeSingleTurn(messages)
+
+ const authorized = wired.authorize.mock.calls[0][0]
+ expect(authorized.request.schemaVersion).toBe('codex-completion-request.v2')
+ expect(authorized.request.messages[0].content).toBe(messages[0].content)
+ expect(parseCodexCompletionRequest(authorized.request).ok).toBe(true)
+ })
+
+ it('restates content from the parts when the caller left them out of sync', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+ const messages = userWithImage()
+ // A stale `content` must never reach the wire as a contradiction of its own
+ // parts: V2 rejects the request outright, so the projection rebuilds it.
+ messages[0].content = 'stale text that no part carries'
+
+ await provider.completeSingleTurn(messages)
+
+ expect(wired.authorize.mock.calls[0][0].request.messages[0].content).toBe('what is on screen?')
+ })
+
+ it.each(METHODS)(
+ 'rejects a signature-only image stub through %s before authorize',
+ async method => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+
+ await expect(
+ invoke(provider, method, userWithImage('iVBORw0KGgo=', 'image/png'))
+ ).rejects.toMatchObject({ name: 'CodexAuthorizeError', code: 'invalid_request' })
+ expect(wired.authorize).not.toHaveBeenCalled()
+ expect(wired.stream).not.toHaveBeenCalled()
+ }
+ )
+
+ it('rejects an over-limit image batch through the shared contract, not a local copy', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', {
+ ...wired,
+ imageInputEnabled: true,
+ } as never)
+ const messages = userWithImage()
+ messages[0].contentParts = [
+ { type: 'text', text: 'what is on screen?' },
+ ...Array.from({ length: 4 }, (_, index) => ({
+ type: 'image' as const,
+ mimeType: 'image/png' as const,
+ data: PNG_2X2_BASE64,
+ source: {
+ kind: 'attachment' as const,
+ attachmentId: `att-${index}`,
+ messageId: 'msg-1',
+ },
+ })),
+ ]
+
+ await expect(provider.completeSingleTurn(messages)).rejects.toMatchObject({
+ name: 'CodexAuthorizeError',
+ code: 'invalid_request',
+ })
+ expect(wired.authorize).not.toHaveBeenCalled()
+ })
+
+ it('keeps a text-only turn on the unchanged V1 request', async () => {
+ const wired = deps()
+ const provider = new CodexSubscriptionProvider('gpt-5.6-luna', wired as never)
+
+ await provider.completeSingleTurn([{ role: 'user', content: 'hi' }])
+
+ const request = wired.authorize.mock.calls[0][0].request
+ expect(request.schemaVersion).toBe('codex-completion-request.v1')
+ expect(request.messages).toEqual([{ role: 'user', content: 'hi' }])
+ expect(JSON.stringify(request)).not.toContain('contentParts')
+ })
})
diff --git a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
index bf699138d..86db35dce 100644
--- a/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
+++ b/mcp-host/src/llm/__tests__/providerAttemptAuthorizer.test.ts
@@ -13,6 +13,46 @@ const validAuthorize = {
}
describe('ProviderAttemptAuthorizer', () => {
+ it('rejects the complete oversized envelope before dispatch', async () => {
+ const fetchFn = vi.fn()
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: 'http://gateway/authorize',
+ readPlatformJwt: () => 'test-jwt',
+ fetchFn,
+ })
+ await expect(
+ authorizer.authorize({
+ request: { content: 'a'.repeat(1048576) },
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ })
+ ).rejects.toMatchObject({ code: 'payload_too_large' })
+ expect(fetchFn).not.toHaveBeenCalled()
+ })
+
+ it('recognizes a non-JSON 413 as a request limit', async () => {
+ const fetchFn = vi.fn(async () => new Response('<h1>Too large</h1>', { status: 413 }))
+ const authorizer = new ProviderAttemptAuthorizer({
+ authorizeUrl: 'http://gateway/authorize',
+ readPlatformJwt: () => 'test-jwt',
+ fetchFn,
+ })
+ await expect(
+ authorizer.authorize({
+ request: {},
+ invocationId: 'inv-1',
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: 1,
+ policyHash: 'b'.repeat(64),
+ })
+ ).rejects.toMatchObject({ code: 'payload_too_large' })
+ expect(fetchFn).toHaveBeenCalledOnce()
+ })
+
it('posts the platform JWT to the server-owned gateway URL', async () => {
const fetchFn = vi.fn().mockResolvedValue({
ok: true,
diff --git a/mcp-host/src/llm/codexLlmProxyClient.ts b/mcp-host/src/llm/codexLlmProxyClient.ts
index 91a5e5aa6..24b622d99 100644
--- a/mcp-host/src/llm/codexLlmProxyClient.ts
+++ b/mcp-host/src/llm/codexLlmProxyClient.ts
@@ -1,3 +1,8 @@
+import {
+ buildCodexProxyEnvelope,
+ parseCodexCompletionRequest,
+} from '@clerum/llm-provider-attempt-contract'
+
export const CODEX_PROXY_COMPLETIONS_PATH = '/internal/runtime/v1/codex/completions'
export class CodexProxyError extends Error {
@@ -72,6 +77,41 @@ export class CodexLlmProxyClient {
},
retryOnUnauthorized: boolean
): Promise<CodexProxyStreamResult> {
+ let body: unknown = {
+ executionTicket: input.executionTicket,
+ requestHash: input.requestHash,
+ request: input.request,
+ ...(input.deadlineMs !== undefined ? { deadlineMs: input.deadlineMs } : {}),
+ }
+ if (
+ (input.request as { schemaVersion?: string } | null)?.schemaVersion ===
+ 'codex-completion-request.v2'
+ ) {
+ const parsed = parseCodexCompletionRequest(input.request)
+ if (!parsed.ok) throw new CodexProxyError('invalid_request', parsed.message, false)
+ if (input.deadlineMs !== undefined && input.deadlineMs !== parsed.value.deadlineMs) {
+ throw new CodexProxyError(
+ 'invalid_request',
+ 'Codex deadline must match the authorized request',
+ false
+ )
+ }
+ const envelope = buildCodexProxyEnvelope({
+ executionTicket: input.executionTicket,
+ requestHash: input.requestHash,
+ request: parsed.value,
+ })
+ if (!envelope.ok) {
+ const code =
+ envelope.code === 'limit'
+ ? 'payload_too_large'
+ : envelope.code === 'request_hash_mismatch'
+ ? 'request_hash_mismatch'
+ : 'invalid_request'
+ throw new CodexProxyError(code, envelope.message, false)
+ }
+ body = envelope.value
+ }
const jwt = this.options.readPlatformJwt()
const fetchFn = this.options.fetchFn ?? fetch
const response = await fetchFn(this.options.runtimeUrl, {
@@ -80,12 +120,7 @@ export class CodexLlmProxyClient {
authorization: `Bearer ${jwt}`,
'content-type': 'application/json',
},
- body: JSON.stringify({
- executionTicket: input.executionTicket,
- requestHash: input.requestHash,
- request: input.request,
- ...(input.deadlineMs !== undefined ? { deadlineMs: input.deadlineMs } : {}),
- }),
+ body: JSON.stringify(body),
signal: input.signal,
})
if (!response.ok) {
@@ -94,8 +129,18 @@ export class CodexLlmProxyClient {
return this.streamOnce(input, false)
}
const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>
- const code = typeof payload.error === 'string' ? payload.error : 'provider_unavailable'
- throw new CodexProxyError(code, `proxy stream failed with ${response.status} (${code})`)
+ const code =
+ response.status === 413
+ ? 'payload_too_large'
+ : typeof payload.error === 'string'
+ ? payload.error
+ : 'provider_unavailable'
+ throw new CodexProxyError(
+ code,
+ code === 'payload_too_large'
+ ? 'Codex request is too large; use fewer or smaller images, or reduce context'
+ : `proxy stream failed with ${response.status} (${code})`
+ )
}
if (!response.body) {
throw new CodexProxyError('provider_unavailable', 'proxy stream had no body')
diff --git a/mcp-host/src/llm/codexSubscription.ts b/mcp-host/src/llm/codexSubscription.ts
index 455e69620..66355e0ad 100644
--- a/mcp-host/src/llm/codexSubscription.ts
+++ b/mcp-host/src/llm/codexSubscription.ts
@@ -1,22 +1,26 @@
import { randomUUID } from 'node:crypto'
import {
- type CodexCompletionRequestV1,
+ type CodexCompletionRequest,
LIMITS,
- hashCodexCompletionRequestV1,
+ hashCodexCompletionRequest,
+ parseCodexCompletionRequest,
} from '@clerum/llm-provider-attempt-contract'
import { LlmErrorCode } from '../core/errors'
import {
CompletionResponse,
ChatMessage as CoreChatMessage,
FinishReason,
+ MessageContentImageSource,
+ MessageContentPart,
ToolCompletionResponse,
ToolDefinition,
+ textContentFromParts,
} from '../core/types'
import { logger } from '../logger'
import { CodexLlmProxyClient, CodexProxyError } from './codexLlmProxyClient'
import { classifyUnknown } from './errorClassification'
import { CodexAuthorizeError, ProviderAttemptAuthorizer } from './providerAttemptAuthorizer'
-import type { LlmProvider } from './registryCore'
+import { type LlmProvider, descriptorFor } from './registryCore'
import type { ClassifiedError, SingleTurnProvider } from './types'
export type CodexAttemptContext = {
@@ -60,6 +64,72 @@ export type CodexSubscriptionDeps = {
authorizer: ProviderAttemptAuthorizer
proxy: CodexLlmProxyClient
attemptContext: (input: { model: string }) => CodexAttemptContext
+ /**
+ * Visual rollout gate (#650). Defaults to FALSE: a request that actually
+ * carries image parts is rejected with `image_input_unsupported` until the
+ * operator clears this model for image input in the registry/config wiring.
+ */
+ imageInputEnabled?: boolean
+}
+
+/**
+ * Pre-authorize rejections. Both are known, non-retryable codes: the request
+ * itself cannot succeed, so neither may trigger a retry or a provider fallback.
+ */
+const CODEX_REQUEST_INVALID = 'invalid_request'
+const CODEX_IMAGE_INPUT_UNSUPPORTED = 'image_input_unsupported'
+const CODEX_IMAGE_SOURCE_INVALID = 'image_source_invalid'
+
+/** One image part as the Codex V2 contract serializes it. */
+type ProjectedImagePart = {
+ type: 'image'
+ mimeType: 'image/jpeg' | 'image/png'
+ data: string
+ source: MessageContentImageSource
+}
+
+/** One message as either contract version serializes it. */
+type ProjectedMessage = {
+ role: CoreChatMessage['role']
+ content: string
+ contentParts?: Array<{ type: 'text'; text: string } | ProjectedImagePart>
+ name?: string
+ toolCallId?: string
+ toolCalls?: Array<{ id: string; name: string; arguments: Record<string, unknown> }>
+}
+
+/**
+ * Provenance for one image part. The shared contract requires a source on every
+ * Codex V2 image and bounds its id charset; this checks only presence and shape,
+ * so the contract stays the single owner of format and size limits.
+ */
+function imageSourceError(detail: string): CodexAuthorizeError {
+ return new CodexAuthorizeError(
+ CODEX_IMAGE_SOURCE_INVALID,
+ `image part has no usable provenance source (${detail}); host producers must attach the attachment or tool call it came from`
+ )
+}
+
+function projectImageSource(
+ part: Extract<MessageContentPart, { type: 'image' }>
+): MessageContentImageSource {
+ const source = part.source
+ if (!source) throw imageSourceError('missing source')
+ if (source.kind === 'attachment') {
+ if (!source.attachmentId?.trim()) throw imageSourceError('empty attachmentId')
+ if (!source.messageId?.trim()) throw imageSourceError('empty messageId')
+ return {
+ kind: 'attachment',
+ attachmentId: source.attachmentId,
+ messageId: source.messageId,
+ }
+ }
+ if (source.kind === 'tool') {
+ if (!source.attachmentId?.trim()) throw imageSourceError('empty attachmentId')
+ if (!source.toolCallId?.trim()) throw imageSourceError('empty toolCallId')
+ return { kind: 'tool', attachmentId: source.attachmentId, toolCallId: source.toolCallId }
+ }
+ throw imageSourceError('unknown source kind')
}
function assertTerminalCodexOutcome(result: {
@@ -88,6 +158,8 @@ function assertTerminalCodexOutcome(result: {
}
export class CodexSubscriptionProvider implements SingleTurnProvider {
+ readonly requiresImageSourceIdentity =
+ descriptorFor('codex-subscription').requiresImageSourceIdentity === true
private nextProviderAttemptIndex = 1
constructor(
@@ -246,7 +318,19 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
throw new CodexProxyError('canceled', 'aborted before authorize', false)
}
const request = this.buildRequest(messages, tools, options)
- const requestHash = hashCodexCompletionRequestV1(request)
+ // Validate the projection before anything is authorized or hashed: the
+ // ticket, the proxy and control-api must all bind the same canonical
+ // request, and an unusable visual payload must fail here as a known,
+ // non-retryable error instead of being dropped on the way out.
+ const parsed = parseCodexCompletionRequest(request)
+ if (!parsed.ok) {
+ throw new CodexAuthorizeError(
+ CODEX_REQUEST_INVALID,
+ `codex completion request rejected: ${parsed.message}`
+ )
+ }
+ const wireRequest = parsed.value
+ const requestHash = hashCodexCompletionRequest(wireRequest)
const context = this.deps.attemptContext({ model: this.model })
if (
!Number.isInteger(context.policyRevision) ||
@@ -258,9 +342,9 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
const providerAttemptIndex = context.providerAttemptIndex ?? this.nextProviderAttemptIndex++
const authorized = await this.deps.authorizer.authorize(
{
- request,
+ request: wireRequest,
requestHash,
- invocationId: context.invocationId ?? request.requestId,
+ invocationId: context.invocationId ?? wireRequest.requestId,
attemptGeneration: context.attemptGeneration ?? 1,
providerAttemptIndex,
policyRevision: context.policyRevision,
@@ -285,7 +369,7 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
const streamed = await this.deps.proxy.stream({
executionTicket: authorized.executionTicket,
requestHash: authorized.requestHash,
- request,
+ request: wireRequest,
signal: options?.signal,
})
return {
@@ -295,63 +379,110 @@ export class CodexSubscriptionProvider implements SingleTurnProvider {
}
}
+ /**
+ * Project the loop's messages onto the wire contract.
+ *
+ * A turn with no parts stays byte-identical to V1. As soon as one message
+ * carries parts the whole request moves to V2, where the text parts are
+ * authoritative for `content` — that is what keeps a redacted (text-only)
+ * message consistent with the contract's equality rule.
+ */
+ private projectMessage(message: CoreChatMessage): ProjectedMessage {
+ const projected: ProjectedMessage = {
+ role: message.role,
+ content: message.content ?? '',
+ ...(message.name ? { name: message.name } : {}),
+ ...(message.tool_call_id ? { toolCallId: message.tool_call_id } : {}),
+ ...(message.role === 'assistant' && message.tool_calls && message.tool_calls.length > 0
+ ? {
+ toolCalls: message.tool_calls.map(call => ({
+ id: call.id,
+ name: call.name,
+ arguments: call.arguments,
+ })),
+ }
+ : {}),
+ }
+ const parts = message.contentParts
+ if (!parts || parts.length === 0) return projected
+ if (message.role !== 'user') {
+ throw new CodexAuthorizeError(
+ CODEX_REQUEST_INVALID,
+ `content parts are only supported on user messages (role=${message.role})`
+ )
+ }
+ if (parts.some(part => part.type === 'image') && this.deps.imageInputEnabled !== true) {
+ throw new CodexAuthorizeError(
+ CODEX_IMAGE_INPUT_UNSUPPORTED,
+ 'Image input is not enabled for this Codex model; remove the attachments or enable image input for the codex-subscription provider'
+ )
+ }
+ const contentParts: NonNullable<ProjectedMessage['contentParts']> = parts.map(part =>
+ part.type === 'text'
+ ? { type: 'text', text: part.text }
+ : {
+ type: 'image',
+ mimeType: part.mimeType,
+ data: part.data,
+ source: projectImageSource(part),
+ }
+ )
+ return { ...projected, content: textContentFromParts(contentParts), contentParts }
+ }
+
private buildRequest(
messages: CoreChatMessage[],
tools: ToolDefinition[] | undefined,
options?: { max_tokens?: number; temperature?: number; tool_choice?: string }
- ): CodexCompletionRequestV1 {
- const request: CodexCompletionRequestV1 = {
- schemaVersion: 'codex-completion-request.v1',
- requestId: randomUUID(),
- idempotencyKey: randomUUID(),
- provider: 'codex-subscription',
- model: this.model,
- messages: messages.map(message => ({
- role: message.role,
- content: message.content ?? '',
- ...(message.name ? { name: message.name } : {}),
- ...(message.tool_call_id ? { toolCallId: message.tool_call_id } : {}),
- ...(message.role === 'assistant' && message.tool_calls && message.tool_calls.length > 0
- ? {
- toolCalls: message.tool_calls.map(call => ({
- id: call.id,
- name: call.name,
- arguments: call.arguments,
- })),
- }
- : {}),
- })),
- }
- if (tools && tools.length > 0) {
+ ): CodexCompletionRequest {
+ const projectedMessages = messages.map(message => this.projectMessage(message))
+ const presentedTools =
+ tools && tools.length > 0
+ ? tools.map(tool => ({
+ name: tool.name,
+ description: tool.description,
+ parameters: tool.parameters,
+ }))
+ : undefined
+ if (presentedTools) {
// Presentation is owned by the agent loop. Preserve its final definitions
// in both the authorized hash and the proxy request; MCP names are not
// permissions and must never be discarded by the provider adapter.
logger.debug(
- { provider: 'codex-subscription', presentedCount: tools.length },
+ { provider: 'codex-subscription', presentedCount: presentedTools.length },
'Tool definitions prepared for authorization'
)
- request.tools = tools.map(tool => ({
- name: tool.name,
- description: tool.description,
- parameters: tool.parameters,
- }))
}
-
- if (
+ const hasGeneration =
options?.temperature !== undefined ||
options?.max_tokens !== undefined ||
- options?.tool_choice
- ) {
- request.generation = {
- ...(options.temperature !== undefined ? { temperature: options.temperature } : {}),
- ...(options.max_tokens !== undefined ? { maxOutputTokens: options.max_tokens } : {}),
- ...(options.tool_choice === 'auto' ||
- options.tool_choice === 'none' ||
- options.tool_choice === 'required'
- ? { toolChoice: options.tool_choice }
- : {}),
- }
+ Boolean(options?.tool_choice)
+ const base: Omit<CodexCompletionRequest, 'schemaVersion'> = {
+ requestId: randomUUID(),
+ idempotencyKey: randomUUID(),
+ provider: 'codex-subscription' as const,
+ model: this.model,
+ messages: projectedMessages,
+ ...(presentedTools ? { tools: presentedTools } : {}),
+ ...(hasGeneration
+ ? {
+ generation: {
+ ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}),
+ ...(options?.max_tokens !== undefined ? { maxOutputTokens: options.max_tokens } : {}),
+ ...(options?.tool_choice === 'auto' ||
+ options?.tool_choice === 'none' ||
+ options?.tool_choice === 'required'
+ ? { toolChoice: options.tool_choice }
+ : {}),
+ },
+ }
+ : {}),
+ }
+ // V1 stays byte-identical for every text-only turn; one message with parts
+ // moves the whole request to V2, which is the only version that carries them.
+ if (!projectedMessages.some(message => message.contentParts !== undefined)) {
+ return { schemaVersion: 'codex-completion-request.v1', ...base }
}
- return request
+ return { schemaVersion: 'codex-completion-request.v2', ...base }
}
}
diff --git a/mcp-host/src/llm/index.ts b/mcp-host/src/llm/index.ts
index 668bf24fc..9ab1079de 100644
--- a/mcp-host/src/llm/index.ts
+++ b/mcp-host/src/llm/index.ts
@@ -2,6 +2,7 @@
* LLM Provider factory.
*/
import { config } from '../config'
+import { logger } from '../logger'
import { ApiKeys, ModelConfig } from '../types'
import { ClaudeProvider } from './claude'
import { CodexLlmProxyClient, resolveCodexProxyRuntimeUrl } from './codexLlmProxyClient'
@@ -22,9 +23,10 @@ export type { ClassifiedError, SingleTurnProvider } from './types'
const DEFAULT_CODEX_AUTHORIZE_GATEWAY =
'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092'
-function createCodexRuntimeDeps(captured?: CodexAttemptContext) {
+function createCodexRuntimeDeps(model: string | undefined, captured?: CodexAttemptContext) {
const gateway = (config.mcpHostGatewayUrl ?? '').trim() || DEFAULT_CODEX_AUTHORIZE_GATEWAY
return {
+ imageInputEnabled: Boolean(model && config.codexImageInputModels.includes(model)),
authorizer: new ProviderAttemptAuthorizer({
authorizeUrl: resolveCodexAuthorizeUrl(gateway),
readPlatformJwt: readCodexPlatformJwt,
@@ -70,7 +72,7 @@ export function createLLMProvider(
const modelName = modelConfig?.name
if (!isLlmProvider(provider)) {
- console.error('[LLM] Unknown provider')
+ logger.error({}, 'Unknown LLM provider')
return null
}
@@ -81,7 +83,7 @@ export function createLLMProvider(
const credentials = keys[provider] ?? {}
for (const slot of descriptorFor(provider).credentialSlots) {
if (slot.required && !credentials[slot.dataKey]) {
- console.error('[LLM] required credential missing from secrets')
+ logger.error({}, 'Required LLM credential is missing')
return null
}
}
@@ -98,11 +100,11 @@ export function createLLMProvider(
credentials,
modelName,
provider === 'codex-subscription'
- ? { codex: createCodexRuntimeDeps(options?.capturedCodexAttemptContext) }
+ ? { codex: createCodexRuntimeDeps(modelName, options?.capturedCodexAttemptContext) }
: undefined
)
} catch (err) {
- console.error('[LLM] failed to construct provider')
+ logger.error({}, 'Failed to construct LLM provider')
return null
}
}
diff --git a/mcp-host/src/llm/providerAttemptAuthorizer.ts b/mcp-host/src/llm/providerAttemptAuthorizer.ts
index 0fd18520b..d6379d05b 100644
--- a/mcp-host/src/llm/providerAttemptAuthorizer.ts
+++ b/mcp-host/src/llm/providerAttemptAuthorizer.ts
@@ -1,4 +1,4 @@
-import { parseAuthorizeAttemptResponse } from '@clerum/llm-provider-attempt-contract'
+import { LIMITS, parseAuthorizeAttemptResponse } from '@clerum/llm-provider-attempt-contract'
export const AUTHORIZE_PATH = '/api/v1/mcp-host/llm/provider-attempts/authorize'
@@ -76,6 +76,13 @@ export class ProviderAttemptAuthorizer {
executionTicket: string
expiresAt: string
}> {
+ const serialized = JSON.stringify(body)
+ if (Buffer.byteLength(serialized, 'utf8') > LIMITS.maxRequestBodyBytes) {
+ throw new CodexAuthorizeError(
+ 'payload_too_large',
+ 'Codex request exceeds 1 MiB; use fewer or smaller images, or reduce context'
+ )
+ }
const jwt = this.options.readPlatformJwt()
if (!jwt) {
throw new CodexAuthorizeError('no_grant', 'platform JWT is missing')
@@ -87,7 +94,7 @@ export class ProviderAttemptAuthorizer {
authorization: `Bearer ${jwt}`,
'content-type': 'application/json',
},
- body: JSON.stringify(body),
+ body: serialized,
...(signal ? { signal } : {}),
})
const payload = (await response.json().catch(() => ({}))) as Record<string, unknown>
@@ -96,8 +103,18 @@ export class ProviderAttemptAuthorizer {
await this.options.refreshOnUnauthorized()
return this.authorizeOnce(body, false, signal)
}
- const code = typeof payload.error === 'string' ? payload.error : 'provider_unavailable'
- throw new CodexAuthorizeError(code, `authorize failed with ${response.status}`)
+ const code =
+ response.status === 413
+ ? 'payload_too_large'
+ : typeof payload.error === 'string'
+ ? payload.error
+ : 'provider_unavailable'
+ throw new CodexAuthorizeError(
+ code,
+ code === 'payload_too_large'
+ ? 'Codex request is too large; use fewer or smaller images, or reduce context'
+ : `authorize failed with ${response.status}`
+ )
}
for (const key of LEAK_KEYS) {
if (key in payload) {
diff --git a/mcp-host/src/llm/registryCore.ts b/mcp-host/src/llm/registryCore.ts
index 83255ff67..643948fe2 100644
--- a/mcp-host/src/llm/registryCore.ts
+++ b/mcp-host/src/llm/registryCore.ts
@@ -52,6 +52,8 @@ import {
export type { CredentialSlot }
export interface CoreProviderDescriptor {
+ /** Requests bind distinct visual source identities, even for repeated bytes. */
+ requiresImageSourceIdentity?: boolean
/** Canonical id, == getProviderType(). */
id: string
authMode: ProviderAuthMode
@@ -90,6 +92,7 @@ export type LlmProvider = LlmProviderId
* `@clerum/llm-providers`.
*/
interface RuntimeProviderFields {
+ requiresImageSourceIdentity?: boolean
defaultModel?: string
/** Base URL for OpenAI-compatible providers (zai, bailian). */
baseURL?: string
@@ -200,7 +203,7 @@ const RUNTIME_FIELDS: Record<LlmProvider, RuntimeProviderFields> = {
// defaultModel is the Azure DEPLOYMENT name the operator expects by default.
azure: { defaultModel: 'gpt-4.1', tokenizer: 'openai' },
// Broker: explicit model required later; no Secret slot and no default.
- 'codex-subscription': { tokenizer: 'fallback' },
+ 'codex-subscription': { tokenizer: 'fallback', requiresImageSourceIdentity: true },
}
// Order = dev auto-detection priority (first present key wins), inherited from
diff --git a/mcp-host/src/llm/types.ts b/mcp-host/src/llm/types.ts
index 4afd896fb..28744f527 100644
--- a/mcp-host/src/llm/types.ts
+++ b/mcp-host/src/llm/types.ts
@@ -63,6 +63,8 @@ export interface ClassifiedError {
* Single-turn LLM transport. No loop, no tool execution.
*/
export interface SingleTurnProvider {
+ /** The request contract binds distinct image sources, even when their bytes repeat. */
+ readonly requiresImageSourceIdentity?: boolean
completeSingleTurn(
messages: CoreChatMessage[],
options?: { max_tokens?: number; temperature?: number; signal?: AbortSignal }
diff --git a/packages/llm-provider-attempt-contract/fixtures/canonical-request-hashes.v2.json b/packages/llm-provider-attempt-contract/fixtures/canonical-request-hashes.v2.json
new file mode 100644
index 000000000..f26238fcf
--- /dev/null
+++ b/packages/llm-provider-attempt-contract/fixtures/canonical-request-hashes.v2.json
@@ -0,0 +1,736 @@
+{
+ "schemaVersion": "codex-completion-request.v2",
+ "notes": "SHA-256 over the validated V2 projection using the same lexicographic stableStringify as V1. Array order (messages, contentParts, toolCalls) is significant. Image parts carry `dataRef` (resolved by index.test.cjs from fixtures/visual-requests.json, whose imageSha256 pins the same bytes); `mutate` applies a documented truncation. The frozen request digest therefore pins those bytes transitively.",
+ "cases": [
+ {
+ "name": "text-only-v2",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-text",
+ "idempotencyKey": "idem-req-v2-text",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "hello"
+ }
+ ]
+ },
+ "sha256": "e2a0c9e0922b0a94d8d94a6d78c55324729a4bab340ed69c7eb0c6622bb48b13"
+ },
+ {
+ "name": "text-and-png-attachment",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-png",
+ "idempotencyKey": "idem-req-v2-png",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Describe the attached image",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "Describe the attached image"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "sha256": "725a6fdf72f9b93a139545e3a33c351f3674aba65ebfbcfb48da72608cb0799e"
+ },
+ {
+ "name": "image-only-jpeg-tool-source",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-jpeg",
+ "idempotencyKey": "idem-req-v2-jpeg",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "",
+ "contentParts": [
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "dataRef": "jpeg",
+ "source": {
+ "kind": "tool",
+ "attachmentId": "att_1700000000001_ff00aa11",
+ "toolCallId": "call_abc123"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "sha256": "a54b4f8e5a530a2791475c7f91273dbdb2038b25f198c221271219565656eb99"
+ },
+ {
+ "name": "text-only-parts-after-compaction",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-compacted",
+ "idempotencyKey": "idem-req-v2-compacted",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "first line\nsecond line",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "first line"
+ },
+ {
+ "type": "text",
+ "text": "second line"
+ }
+ ]
+ }
+ ]
+ },
+ "sha256": "3e1cd9ef11a6ad3f63525a50cbb2c06403c14c25664c5ce0ecbdac2b5f31623e"
+ },
+ {
+ "name": "multi-image-order-significant",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-multi",
+ "idempotencyKey": "idem-req-v2-multi",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "compare",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "compare"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ },
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "dataRef": "jpeg",
+ "source": {
+ "kind": "tool",
+ "attachmentId": "att_1700000000001_ff00aa11",
+ "toolCallId": "call_abc123"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "distinctRequest": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-multi",
+ "idempotencyKey": "idem-req-v2-multi",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "compare",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "compare"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "dataRef": "jpeg",
+ "source": {
+ "kind": "tool",
+ "attachmentId": "att_1700000000001_ff00aa11",
+ "toolCallId": "call_abc123"
+ }
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "sha256": "78ad94e65736575e1a4dc269fe0149d9fa5e2d054b961839ea27c388be2afb77"
+ },
+ {
+ "name": "source-identity-is-hashed",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-png",
+ "idempotencyKey": "idem-req-v2-png",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Describe the attached image",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "Describe the attached image"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "distinctRequest": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-png",
+ "idempotencyKey": "idem-req-v2-png",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Describe the attached image",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "Describe the attached image"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-2"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "sha256": "725a6fdf72f9b93a139545e3a33c351f3674aba65ebfbcfb48da72608cb0799e"
+ }
+ ],
+ "rejects": [
+ {
+ "name": "content-parts-contradict-content",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r1",
+ "idempotencyKey": "idem-req-v2-r1",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "different text",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "actual text"
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].content must equal the text parts joined by '\\n'"
+ },
+ {
+ "name": "content-parts-join-uses-newline",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r2",
+ "idempotencyKey": "idem-req-v2-r2",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "ab",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "a"
+ },
+ {
+ "type": "text",
+ "text": "b"
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].content must equal the text parts joined by '\\n'"
+ },
+ {
+ "name": "content-parts-empty-array",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r3",
+ "idempotencyKey": "idem-req-v2-r3",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "",
+ "contentParts": []
+ }
+ ]
+ },
+ "message": "messages[0].contentParts must be a non-empty array"
+ },
+ {
+ "name": "content-parts-on-assistant",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r4",
+ "idempotencyKey": "idem-req-v2-r4",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "assistant",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts is only allowed on user"
+ },
+ {
+ "name": "image-only-with-nonempty-content",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r5",
+ "idempotencyKey": "idem-req-v2-r5",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "hi",
+ "contentParts": [
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].content must equal the text parts joined by '\\n'"
+ },
+ {
+ "name": "unknown-part-type",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r6",
+ "idempotencyKey": "idem-req-v2-r6",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "audio",
+ "text": "x"
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[0].type is not allowed"
+ },
+ {
+ "name": "unknown-part-field",
+ "reason": "unknown-field",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r7",
+ "idempotencyKey": "idem-req-v2-r7",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x",
+ "extra": 1
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[0] rejects field 'extra'"
+ },
+ {
+ "name": "image-mime-not-allowed",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r8",
+ "idempotencyKey": "idem-req-v2-r8",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/webp",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1].mimeType is not allowed"
+ },
+ {
+ "name": "image-data-not-canonical-base64",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r9",
+ "idempotencyKey": "idem-req-v2-r9",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "data": "iVBORw0KGgo",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1]: image data must be canonical base64"
+ },
+ {
+ "name": "png-signature-only",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r10",
+ "idempotencyKey": "idem-req-v2-r10",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "data": "iVBORw0KGgo=",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1]: PNG has no IHDR"
+ },
+ {
+ "name": "png-ihdr-only-truncated",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r11",
+ "idempotencyKey": "idem-req-v2-r11",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "mutate": "truncate-after-ihdr",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1]: PNG has no IDAT image data"
+ },
+ {
+ "name": "jpeg-eoi-missing",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r12",
+ "idempotencyKey": "idem-req-v2-r12",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "dataRef": "jpeg",
+ "mutate": "strip-eoi",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1]: JPEG EOI marker is missing"
+ },
+ {
+ "name": "image-missing-source",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r13",
+ "idempotencyKey": "idem-req-v2-r13",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png"
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1].source must be an object"
+ },
+ {
+ "name": "image-source-unknown-kind",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r14",
+ "idempotencyKey": "idem-req-v2-r14",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "web",
+ "attachmentId": "a1",
+ "messageId": "m1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1].source.kind is not allowed"
+ },
+ {
+ "name": "image-source-extra-field",
+ "reason": "unknown-field",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r15",
+ "idempotencyKey": "idem-req-v2-r15",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "a1",
+ "messageId": "m1",
+ "toolCallId": "t1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1].source rejects field 'toolCallId'"
+ },
+ {
+ "name": "image-source-id-too-long",
+ "reason": "invalid",
+ "request": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-r16",
+ "idempotencyKey": "idem-req-v2-r16",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "x",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "x"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "dataRef": "png",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "messageId": "m1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "message": "messages[0].contentParts[1].source.attachmentId is invalid"
+ }
+ ]
+}
diff --git a/packages/llm-provider-attempt-contract/fixtures/visual-requests.json b/packages/llm-provider-attempt-contract/fixtures/visual-requests.json
new file mode 100644
index 000000000..7e043b9ff
--- /dev/null
+++ b/packages/llm-provider-attempt-contract/fixtures/visual-requests.json
@@ -0,0 +1,97 @@
+{
+ "schemaVersion": "codex-completion-request.v2",
+ "notes": "Reusable V2 requests with real, decodable 2x2 images and inline base64, ready to send as-is. PNG: 2x2 RGB truecolor built with zlib.deflateSync and real CRCs (75 bytes). JPEG: 2x2 baseline encoded by the macOS ImageIO `sips` encoder (834 bytes). Both decoded as 2x2 through `sips -g pixelWidth -g pixelHeight -g format` at capture time. The contract validates framing and headers only and does not claim full decode.",
+ "imageSha256": {
+ "png": "becbba8bdf9d37f1628e1f6f587111a4a29d7e5b730582e77d518ddb00bebd8e",
+ "jpeg": "0e6db313e0bb29c0ed8937efd0c833eec96e15f5da4449640201783286cf8e1a"
+ },
+ "expectedDimensions": {
+ "png": {
+ "width": 2,
+ "height": 2
+ },
+ "jpeg": {
+ "width": 2,
+ "height": 2
+ }
+ },
+ "png": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-png",
+ "idempotencyKey": "idem-req-v2-png",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Describe the attached image",
+ "contentParts": [
+ {
+ "type": "text",
+ "text": "Describe the attached image"
+ },
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "data": "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAEklEQVR42mP4z8DAwAAi/gMJABvyA/23MHTXAAAAAElFTkSuQmCC",
+ "source": {
+ "kind": "attachment",
+ "attachmentId": "att_1700000000000_ab12cd34",
+ "messageId": "msg-1"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "jpeg": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-jpeg",
+ "idempotencyKey": "idem-req-v2-jpeg",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "",
+ "contentParts": [
+ {
+ "type": "image",
+ "mimeType": "image/jpeg",
+ "data": "/9j/4AAQSkZJRgABAQAASABIAAD/4QBMRXhpZgAATU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAAqADAAQAAAABAAAAAgAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgAAgACAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAgICAgICAwICAwUDAwMFBgUFBQUGCAYGBgYGCAoICAgICAgKCgoKCgoKCgwMDAwMDA4ODg4ODw8PDw8PDw8PD//bAEMBAgICBAQEBwQEBxALCQsQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEP/dAAQAAf/aAAwDAQACEQMRAD8A+BtH0fSLvSLG7u7GCaeaCJ5JHiVnd2UFmZiMkk8knrWl/wAI/oH/AEDbb/vyn+FHh7/kAab/ANe0P/oArXr+qD+1D//Z",
+ "source": {
+ "kind": "tool",
+ "attachmentId": "att_1700000000001_ff00aa11",
+ "toolCallId": "call_abc123"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "pngImageOnly": {
+ "schemaVersion": "codex-completion-request.v2",
+ "requestId": "req-v2-png-only",
+ "idempotencyKey": "idem-req-v2-png-only",
+ "provider": "codex-subscription",
+ "model": "gpt-5.1",
+ "messages": [
+ {
+ "role": "user",
+ "content": "",
+ "contentParts": [
+ {
+ "type": "image",
+ "mimeType": "image/png",
+ "data": "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAEklEQVR42mP4z8DAwAAi/gMJABvyA/23MHTXAAAAAElFTkSuQmCC",
+ "source": {
+ "kind": "tool",
+ "attachmentId": "att_1700000000001_ff00aa11",
+ "toolCallId": "call_abc123"
+ }
+ }
+ ]
+ }
+ ]
+ }
+}
diff --git a/packages/llm-provider-attempt-contract/index.cjs b/packages/llm-provider-attempt-contract/index.cjs
index fa7e9bad3..c431a3f5a 100644
--- a/packages/llm-provider-attempt-contract/index.cjs
+++ b/packages/llm-provider-attempt-contract/index.cjs
@@ -7,7 +7,10 @@
const { createHash } = require('node:crypto')
+const { VISUAL_LIMITS, inspectVisualImage } = require('./visualPayload.cjs')
+
const SCHEMA_VERSION = 'codex-completion-request.v1'
+const SCHEMA_VERSION_V2 = 'codex-completion-request.v2'
const RECEIPT_SCHEMA_VERSION = 'codex-attempt-receipt.v1'
const PROVIDER_ID = 'codex-subscription'
const TICKET_TYP = 'codex-execution-ticket'
@@ -45,6 +48,26 @@ const TOOL_CALL_KEYS = new Set(['id', 'name', 'arguments'])
const TOOL_KEYS = new Set(['name', 'description', 'parameters'])
const GENERATION_KEYS = new Set(['temperature', 'maxOutputTokens', 'toolChoice'])
const HINT_KEYS = new Set(['promptCacheKey'])
+
+/**
+ * V2 (codex-completion-request.v2) adds typed visual parts to user messages.
+ * The root field set is deliberately the same closed set as V1: V2 must not
+ * become a looser schema at the root.
+ */
+const MESSAGE_KEYS_V2 = new Set([
+ 'role',
+ 'content',
+ 'contentParts',
+ 'name',
+ 'toolCallId',
+ 'toolCalls',
+])
+const CONTENT_PART_TEXT_KEYS = new Set(['type', 'text'])
+const CONTENT_PART_IMAGE_KEYS = new Set(['type', 'mimeType', 'data', 'source'])
+const IMAGE_SOURCE_ATTACHMENT_KEYS = new Set(['kind', 'attachmentId', 'messageId'])
+const IMAGE_SOURCE_TOOL_KEYS = new Set(['kind', 'attachmentId', 'toolCallId'])
+const ENVELOPE_KEYS = new Set(['executionTicket', 'requestHash', 'request'])
+
const CLAIMS_KEYS = new Set([
'jti',
'typ',
@@ -65,13 +88,14 @@ const CLAIMS_KEYS = new Set([
'connectionRevision',
'connectionId',
])
-const AUTHORIZE_KEYS = new Set([
+const AUTHORIZE_KEYS = new Set(['providerAttemptId', 'requestHash', 'executionTicket', 'expiresAt'])
+const RECEIPT_KEYS = new Set([
+ 'schemaVersion',
'providerAttemptId',
'requestHash',
- 'executionTicket',
- 'expiresAt',
+ 'outcome',
+ 'usage',
])
-const RECEIPT_KEYS = new Set(['schemaVersion', 'providerAttemptId', 'requestHash', 'outcome', 'usage'])
const USAGE_KEYS = new Set(['inputTokens', 'outputTokens'])
function fail(code, message) {
@@ -165,7 +189,14 @@ function assertFiniteTree(value, label) {
return null
}
-function parseMessages(raw) {
+/**
+ * One message parser for every request version. `messageKeys` is the closed
+ * field set for that version (V2 adds `contentParts`); a version without that
+ * key rejects the field as unknown before this function can project it.
+ * `visualBudget` is a single request-scoped accumulator: image count and total
+ * image bytes bound the entire request, not one message.
+ */
+function parseMessages(raw, messageKeys, visualBudget) {
if (!Array.isArray(raw) || raw.length === 0) {
return fail('invalid', 'messages must be a non-empty array')
}
@@ -175,56 +206,66 @@ function parseMessages(raw) {
const messages = []
for (let i = 0; i < raw.length; i++) {
const item = raw[i]
- if (!isPlainObject(item)) return fail('invalid', `messages[${i}] must be an object`)
- const extra = rejectUnknown(item, MESSAGE_KEYS, `messages[${i}]`)
+ const label = `messages[${i}]`
+ if (!isPlainObject(item)) return fail('invalid', `${label} must be an object`)
+ const extra = rejectUnknown(item, messageKeys, label)
if (extra) return extra
- if (!MESSAGE_ROLES.has(item.role)) return fail('invalid', `messages[${i}].role is not allowed`)
- if (typeof item.content !== 'string') return fail('invalid', `messages[${i}].content must be a string`)
+ if (!MESSAGE_ROLES.has(item.role)) return fail('invalid', `${label}.role is not allowed`)
+ if (typeof item.content !== 'string')
+ return fail('invalid', `${label}.content must be a string`)
const message = { role: item.role, content: item.content }
if (item.name !== undefined) {
- if (!isToolName(item.name)) return fail('invalid', `messages[${i}].name is invalid`)
+ if (!isToolName(item.name)) return fail('invalid', `${label}.name is invalid`)
message.name = item.name
}
if (item.toolCallId !== undefined) {
- if (!isBoundedId(item.toolCallId)) return fail('invalid', `messages[${i}].toolCallId is invalid`)
+ if (!isBoundedId(item.toolCallId)) return fail('invalid', `${label}.toolCallId is invalid`)
message.toolCallId = item.toolCallId
}
if (item.toolCalls !== undefined) {
if (item.role !== 'assistant') {
- return fail('invalid', `messages[${i}].toolCalls is only allowed on assistant`)
+ return fail('invalid', `${label}.toolCalls is only allowed on assistant`)
}
if (!Array.isArray(item.toolCalls) || item.toolCalls.length === 0) {
- return fail('invalid', `messages[${i}].toolCalls must be a non-empty array`)
+ return fail('invalid', `${label}.toolCalls must be a non-empty array`)
}
if (item.toolCalls.length > LIMITS.maxToolCalls) {
- return fail('limit', `messages[${i}].toolCalls exceed ${LIMITS.maxToolCalls}`)
+ return fail('limit', `${label}.toolCalls exceed ${LIMITS.maxToolCalls}`)
}
const toolCalls = []
for (let j = 0; j < item.toolCalls.length; j++) {
const call = item.toolCalls[j]
if (!isPlainObject(call)) {
- return fail('invalid', `messages[${i}].toolCalls[${j}] must be an object`)
+ return fail('invalid', `${label}.toolCalls[${j}] must be an object`)
}
- const callExtra = rejectUnknown(call, TOOL_CALL_KEYS, `messages[${i}].toolCalls[${j}]`)
+ const callExtra = rejectUnknown(call, TOOL_CALL_KEYS, `${label}.toolCalls[${j}]`)
if (callExtra) return callExtra
if (!isBoundedId(call.id)) {
- return fail('invalid', `messages[${i}].toolCalls[${j}].id is invalid`)
+ return fail('invalid', `${label}.toolCalls[${j}].id is invalid`)
}
if (!isToolName(call.name)) {
- return fail('invalid', `messages[${i}].toolCalls[${j}].name is invalid`)
+ return fail('invalid', `${label}.toolCalls[${j}].name is invalid`)
}
if (!isPlainObject(call.arguments)) {
- return fail('invalid', `messages[${i}].toolCalls[${j}].arguments must be an object`)
+ return fail('invalid', `${label}.toolCalls[${j}].arguments must be an object`)
}
- const finiteArgs = assertFiniteTree(
- call.arguments,
- `messages[${i}].toolCalls[${j}].arguments`
- )
+ const finiteArgs = assertFiniteTree(call.arguments, `${label}.toolCalls[${j}].arguments`)
if (finiteArgs) return finiteArgs
toolCalls.push({ id: call.id, name: call.name, arguments: call.arguments })
}
message.toolCalls = toolCalls
}
+ if (item.contentParts !== undefined) {
+ if (item.role !== 'user') {
+ return fail('invalid', `${label}.contentParts is only allowed on user`)
+ }
+ const parts = parseContentParts(item.contentParts, `${label}.contentParts`, visualBudget)
+ if (!parts.ok) return parts
+ if (parts.value.text !== item.content) {
+ return fail('invalid', `${label}.content must equal the text parts joined by '\\n'`)
+ }
+ message.contentParts = parts.value.parts
+ }
messages.push(message)
}
return ok(messages)
@@ -309,7 +350,13 @@ function parseTransportHints(raw) {
return ok({ promptCacheKey: raw.promptCacheKey })
}
-function parseCodexCompletionRequestV1(input) {
+/**
+ * One root parser for every request version: the same closed root field set,
+ * the same ordered checks and the same projection. Only the schema version
+ * literal and the message field set differ, so a new version cannot become a
+ * looser root by accident.
+ */
+function parseCodexCompletionRequestRoot(input, schemaVersion, messageKeys) {
if (!isPlainObject(input)) return fail('invalid', 'request must be an object')
const encoded = Buffer.byteLength(JSON.stringify(input), 'utf8')
if (encoded > LIMITS.maxRequestBodyBytes) {
@@ -317,15 +364,15 @@ function parseCodexCompletionRequestV1(input) {
}
const extra = rejectUnknown(input, ROOT_KEYS, 'request')
if (extra) return extra
- if (input.schemaVersion !== SCHEMA_VERSION) {
- return fail('invalid', 'schemaVersion is not codex-completion-request.v1')
+ if (input.schemaVersion !== schemaVersion) {
+ return fail('invalid', `schemaVersion is not ${schemaVersion}`)
}
if (!isBoundedId(input.requestId)) return fail('invalid', 'requestId is invalid')
if (!isBoundedId(input.idempotencyKey)) return fail('invalid', 'idempotencyKey is invalid')
if (input.provider !== PROVIDER_ID) return fail('invalid', 'provider must be codex-subscription')
if (!isBoundedId(input.model)) return fail('invalid', 'model is invalid')
- const messages = parseMessages(input.messages)
+ const messages = parseMessages(input.messages, messageKeys, { images: 0, totalBytes: 0 })
if (!messages.ok) return messages
const tools = parseTools(input.tools)
if (!tools.ok) return tools
@@ -350,7 +397,7 @@ function parseCodexCompletionRequestV1(input) {
}
const projected = {
- schemaVersion: SCHEMA_VERSION,
+ schemaVersion,
requestId: input.requestId,
idempotencyKey: input.idempotencyKey,
provider: PROVIDER_ID,
@@ -364,10 +411,179 @@ function parseCodexCompletionRequestV1(input) {
return ok(Object.freeze(projected))
}
+function parseCodexCompletionRequestV1(input) {
+ return parseCodexCompletionRequestRoot(input, SCHEMA_VERSION, MESSAGE_KEYS)
+}
+
function hashCodexCompletionRequestV1(request) {
return createHash('sha256').update(stableStringify(request)).digest('hex')
}
+/**
+ * Minimal image provenance for V2. Closed union: an attachment that arrived
+ * with a user message, or one produced by a tool call. It is hashed with the
+ * rest of the part and never becomes model text.
+ *
+ * Identifier shape reuses the existing bounded ID pattern. Evidence from the
+ * current producers: attachment ids are `att_<epoch>_<rand>`
+ * (mcp-host/src/core/tools/desktop/screenshotUtil.ts), channel message ids are
+ * values such as `telegram:tg-chat-1:42` or `1700000001.000001`
+ * (mcp-host/src/workflow/providerWorkflowCallerContextClient.test.ts), and
+ * tool-call ids come from the assistant tool call. All fit
+ * `[A-Za-z0-9._:/-]{1,128}`; no producer needs a different alphabet.
+ */
+function parseImageSource(raw, label) {
+ if (!isPlainObject(raw)) return fail('invalid', `${label} must be an object`)
+ if (raw.kind !== 'attachment' && raw.kind !== 'tool') {
+ return fail('invalid', `${label}.kind is not allowed`)
+ }
+ const fromAttachment = raw.kind === 'attachment'
+ const extra = rejectUnknown(
+ raw,
+ fromAttachment ? IMAGE_SOURCE_ATTACHMENT_KEYS : IMAGE_SOURCE_TOOL_KEYS,
+ label
+ )
+ if (extra) return extra
+ if (!isBoundedId(raw.attachmentId)) return fail('invalid', `${label}.attachmentId is invalid`)
+ const ownerKey = fromAttachment ? 'messageId' : 'toolCallId'
+ if (!isBoundedId(raw[ownerKey])) return fail('invalid', `${label}.${ownerKey} is invalid`)
+ if (fromAttachment) {
+ return ok({ kind: 'attachment', attachmentId: raw.attachmentId, messageId: raw.messageId })
+ }
+ return ok({ kind: 'tool', attachmentId: raw.attachmentId, toolCallId: raw.toolCallId })
+}
+
+/**
+ * Ordered content parts. Order is significant and preserved verbatim; nothing
+ * is merged, reordered or dropped. Text parts must agree with the message
+ * `content` string, so the textual projection and the part projection cannot
+ * contradict each other. Parts are optional and may be text-only (a compacted
+ * message that no longer carries an image is valid); images are bounded by
+ * count, per-image bytes, total bytes, dimension and pixel budgets.
+ */
+function parseContentParts(raw, label, visualBudget) {
+ if (!Array.isArray(raw) || raw.length === 0) {
+ return fail('invalid', `${label} must be a non-empty array`)
+ }
+ const parts = []
+ const textBits = []
+ for (let i = 0; i < raw.length; i++) {
+ const part = raw[i]
+ const partLabel = `${label}[${i}]`
+ if (!isPlainObject(part)) return fail('invalid', `${partLabel} must be an object`)
+ if (part.type === 'text') {
+ const extra = rejectUnknown(part, CONTENT_PART_TEXT_KEYS, partLabel)
+ if (extra) return extra
+ if (typeof part.text !== 'string') {
+ return fail('invalid', `${partLabel}.text must be a string`)
+ }
+ parts.push({ type: 'text', text: part.text })
+ textBits.push(part.text)
+ continue
+ }
+ if (part.type === 'image') {
+ const extra = rejectUnknown(part, CONTENT_PART_IMAGE_KEYS, partLabel)
+ if (extra) return extra
+ if (part.mimeType !== 'image/jpeg' && part.mimeType !== 'image/png') {
+ return fail('invalid', `${partLabel}.mimeType is not allowed`)
+ }
+ const source = parseImageSource(part.source, `${partLabel}.source`)
+ if (!source.ok) return source
+ const image = inspectVisualImage({ mimeType: part.mimeType, data: part.data })
+ if (!image.ok) return fail(image.code, `${partLabel}: ${image.message}`)
+ // Request-scoped budgets: history can carry more messages than one, so a
+ // per-message counter would let a request exceed the declared limits.
+ visualBudget.images += 1
+ if (visualBudget.images > VISUAL_LIMITS.maxImages) {
+ return fail('limit', `request exceeds ${VISUAL_LIMITS.maxImages} images`)
+ }
+ visualBudget.totalBytes += image.value.bytes
+ if (visualBudget.totalBytes > VISUAL_LIMITS.maxTotalImageBytes) {
+ return fail(
+ 'limit',
+ `request exceeds ${VISUAL_LIMITS.maxTotalImageBytes} total image bytes`
+ )
+ }
+ parts.push({
+ type: 'image',
+ mimeType: part.mimeType,
+ data: part.data,
+ source: source.value,
+ })
+ continue
+ }
+ return fail('invalid', `${partLabel}.type is not allowed`)
+ }
+ return ok({ parts, text: textBits.join('\n') })
+}
+
+/**
+ * V2 root: the shared root parser with the V2 schema version and the message
+ * field set that additionally allows `contentParts`.
+ */
+function parseCodexCompletionRequestV2(input) {
+ return parseCodexCompletionRequestRoot(input, SCHEMA_VERSION_V2, MESSAGE_KEYS_V2)
+}
+
+/**
+ * Single dispatcher for every supported version. It only selects the version
+ * and defers all validation to that version's parser, so an unknown or missing
+ * schemaVersion fails closed instead of falling through to a loose path.
+ */
+function parseCodexCompletionRequest(input) {
+ if (!isPlainObject(input)) return fail('invalid', 'request must be an object')
+ if (input.schemaVersion === SCHEMA_VERSION) return parseCodexCompletionRequestV1(input)
+ if (input.schemaVersion === SCHEMA_VERSION_V2) return parseCodexCompletionRequestV2(input)
+ return fail(
+ 'invalid',
+ 'schemaVersion must be codex-completion-request.v1 or codex-completion-request.v2'
+ )
+}
+
+function hashCodexCompletionRequest(request) {
+ return createHash('sha256').update(stableStringify(request)).digest('hex')
+}
+
+/**
+ * Exact proxy request envelope shared by the authorizer and the proxy.
+ *
+ * The outer `deadlineMs` is never emitted: a V2 request carries its deadline in
+ * `request.deadlineMs`, already bound by the request hash, and a caller must
+ * not be able to add an independent deadline after the envelope was measured.
+ *
+ * The returned envelope is re-parsed and re-hashed here, so it cannot carry a
+ * request the contract rejects or a hash that disagrees with the request it
+ * carries. Its exact UTF-8 byte length (JSON.stringify minus whitespace, the
+ * same serialization the transport sends) must fit `LIMITS.maxRequestBodyBytes`
+ * — the default the proxy enforces through CODEX_LLM_PROXY_MAX_BODY_BYTES. A
+ * deployment that lowers that proxy limit below the contract limit is not
+ * covered by this measurement.
+ */
+function buildCodexProxyEnvelope(input) {
+ if (!isPlainObject(input)) return fail('invalid', 'proxy envelope must be an object')
+ const extra = rejectUnknown(input, ENVELOPE_KEYS, 'proxy envelope')
+ if (extra) return extra
+ const ticket = input.executionTicket
+ if (typeof ticket !== 'string' || ticket.length < 8 || /[\p{Cc}\p{Cs}]/u.test(ticket)) {
+ return fail('invalid', 'executionTicket is invalid')
+ }
+ if (typeof input.requestHash !== 'string' || !SHA256_HEX.test(input.requestHash)) {
+ return fail('invalid', 'requestHash must be a SHA-256 hex digest')
+ }
+ const parsed = parseCodexCompletionRequest(input.request)
+ if (!parsed.ok) return parsed
+ const requestHash = hashCodexCompletionRequest(parsed.value)
+ if (requestHash !== input.requestHash) {
+ return fail('request_hash_mismatch', 'requestHash does not match the request')
+ }
+ const envelope = { executionTicket: ticket, requestHash, request: parsed.value }
+ const encoded = Buffer.byteLength(JSON.stringify(envelope), 'utf8')
+ if (encoded > LIMITS.maxRequestBodyBytes) {
+ return fail('limit', 'proxy envelope exceeds maxRequestBodyBytes')
+ }
+ return ok(Object.freeze(envelope))
+}
+
/**
* Grant-binding digest for a Codex authorize attempt. Must match
* control-api's expected hash: SHA-256 of lexicographic stableStringify of
@@ -401,7 +617,12 @@ function requireHex64(obj, key) {
function requireInt(obj, key, min) {
const value = obj[key]
- if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isInteger(value) || value < min) {
+ if (
+ typeof value !== 'number' ||
+ !Number.isFinite(value) ||
+ !Number.isInteger(value) ||
+ value < min
+ ) {
return fail('non-finite', `${key} must be a finite integer`)
}
return null
@@ -411,7 +632,15 @@ function parseCodexExecutionTicketClaims(input) {
if (!isPlainObject(input)) return fail('invalid', 'ticket claims must be an object')
const extra = rejectUnknown(input, CLAIMS_KEYS, 'ticket claims')
if (extra) return extra
- for (const key of ['jti', 'sub', 'hostRef', 'invocationId', 'providerAttemptId', 'budgetReservationId', 'model']) {
+ for (const key of [
+ 'jti',
+ 'sub',
+ 'hostRef',
+ 'invocationId',
+ 'providerAttemptId',
+ 'budgetReservationId',
+ 'model',
+ ]) {
const bad = requireId(input, key)
if (bad) return bad
}
@@ -517,13 +746,19 @@ function parseCodexAttemptReceiptV1(input) {
module.exports = {
SCHEMA_VERSION,
+ SCHEMA_VERSION_V2,
RECEIPT_SCHEMA_VERSION,
PROVIDER_ID,
TICKET_TYP,
LIMITS,
+ VISUAL_LIMITS,
stableStringify,
parseCodexCompletionRequestV1,
+ parseCodexCompletionRequestV2,
+ parseCodexCompletionRequest,
hashCodexCompletionRequestV1,
+ hashCodexCompletionRequest,
+ buildCodexProxyEnvelope,
computeCodexPolicyHash,
parseCodexExecutionTicketClaims,
parseAuthorizeAttemptResponse,
diff --git a/packages/llm-provider-attempt-contract/index.d.ts b/packages/llm-provider-attempt-contract/index.d.ts
index 17acd19f9..713140707 100644
--- a/packages/llm-provider-attempt-contract/index.d.ts
+++ b/packages/llm-provider-attempt-contract/index.d.ts
@@ -6,9 +6,16 @@
* Hashing uses the same lexicographic stableStringify semantics as
* control-api/src/utils/stableStringify.ts. Non-finite numbers are rejected
* before hashing (stringify would otherwise coerce them to null).
+ *
+ * Two request versions share one dispatcher and one hash:
+ * `codex-completion-request.v1` (text messages, unchanged) and
+ * `codex-completion-request.v2` (the same closed root plus optional ordered
+ * `contentParts` on user messages). V2 shares the V1 root field set on
+ * purpose: it is not a looser schema.
*/
export declare const SCHEMA_VERSION: 'codex-completion-request.v1'
+export declare const SCHEMA_VERSION_V2: 'codex-completion-request.v2'
export declare const RECEIPT_SCHEMA_VERSION: 'codex-attempt-receipt.v1'
export declare const PROVIDER_ID: 'codex-subscription'
export declare const TICKET_TYP: 'codex-execution-ticket'
@@ -22,6 +29,19 @@ export declare const LIMITS: {
readonly maxIdLength: 128
}
+/**
+ * Conservative local safety/product budgets for V2 image parts. These are
+ * deliberately not upstream facts: the frozen ChatGPT endpoint is not
+ * certified by these numbers.
+ */
+export declare const VISUAL_LIMITS: {
+ readonly maxImages: 3
+ readonly maxImageBytes: 524288
+ readonly maxTotalImageBytes: 655360
+ readonly maxImageDimension: 8192
+ readonly maxImagePixels: 16777216
+}
+
export type ContractResult<T> =
| { ok: true; value: T }
| { ok: false; code: string; message: string }
@@ -42,6 +62,46 @@ export interface CodexMessageV1 {
toolCalls?: CodexAssistantToolCallV1[]
}
+export interface CodexMessagePartTextV2 {
+ type: 'text'
+ text: string
+}
+
+/**
+ * Minimal provenance for a V2 image part. Closed union: an attachment that
+ * arrived with a user message, or one produced by a tool call. It is hashed
+ * with the rest of the part and never becomes model text.
+ */
+export type CodexImageSourceV2 =
+ | { kind: 'attachment'; attachmentId: string; messageId: string }
+ | { kind: 'tool'; attachmentId: string; toolCallId: string }
+
+export interface CodexMessagePartImageV2 {
+ type: 'image'
+ mimeType: 'image/jpeg' | 'image/png'
+ /** Strict canonical base64 of a structurally valid PNG/JPEG container. */
+ data: string
+ source: CodexImageSourceV2
+}
+
+export type CodexMessagePartV2 = CodexMessagePartTextV2 | CodexMessagePartImageV2
+
+/**
+ * V2 message. `contentParts` is optional and only allowed on user messages.
+ * When present, the text parts joined with `\n` must equal `content`
+ * (an image-only message therefore carries `content: ''`), so the textual and
+ * part projections cannot contradict each other. A parts array may be
+ * text-only after compaction; it does not have to keep an image.
+ */
+export interface CodexMessageV2 {
+ role: CodexMessageRole
+ content: string
+ contentParts?: CodexMessagePartV2[]
+ name?: string
+ toolCallId?: string
+ toolCalls?: CodexAssistantToolCallV1[]
+}
+
export interface CodexToolDefinitionV1 {
name: string
description: string
@@ -73,6 +133,32 @@ export interface CodexCompletionRequestV1 {
transportHints?: CodexTransportHintsV1
}
+export interface CodexCompletionRequestV2 {
+ schemaVersion: 'codex-completion-request.v2'
+ requestId: string
+ idempotencyKey: string
+ provider: 'codex-subscription'
+ model: string
+ messages: CodexMessageV2[]
+ tools?: CodexToolDefinitionV1[]
+ generation?: CodexGenerationOptionsV1
+ deadlineMs?: number
+ transportHints?: CodexTransportHintsV1
+}
+
+export type CodexCompletionRequest = CodexCompletionRequestV1 | CodexCompletionRequestV2
+
+/**
+ * Exact body the control-api authorizer hands to the Codex proxy. The outer
+ * `deadlineMs` is never emitted: a V2 request carries its deadline in
+ * `request.deadlineMs`, already bound by the request hash.
+ */
+export interface CodexProxyEnvelope {
+ executionTicket: string
+ requestHash: string
+ request: CodexCompletionRequest
+}
+
export type CodexExecutionTicketClaims = {
jti: string
typ: 'codex-execution-ticket'
@@ -135,9 +221,19 @@ export declare function stableStringify(value: unknown): string
export declare function parseCodexCompletionRequestV1(
input: unknown
): ContractResult<CodexCompletionRequestV1>
-export declare function hashCodexCompletionRequestV1(
- request: CodexCompletionRequestV1
-): string
+export declare function parseCodexCompletionRequestV2(
+ input: unknown
+): ContractResult<CodexCompletionRequestV2>
+export declare function parseCodexCompletionRequest(
+ input: unknown
+): ContractResult<CodexCompletionRequest>
+export declare function hashCodexCompletionRequestV1(request: CodexCompletionRequestV1): string
+export declare function hashCodexCompletionRequest(request: CodexCompletionRequest): string
+export declare function buildCodexProxyEnvelope(input: {
+ executionTicket: string
+ requestHash: string
+ request: CodexCompletionRequest
+}): ContractResult<CodexProxyEnvelope>
export declare function computeCodexPolicyHash(input: {
model: string
catalogRevision: number
diff --git a/packages/llm-provider-attempt-contract/index.test.cjs b/packages/llm-provider-attempt-contract/index.test.cjs
index 3b9982d90..bf762ea90 100644
--- a/packages/llm-provider-attempt-contract/index.test.cjs
+++ b/packages/llm-provider-attempt-contract/index.test.cjs
@@ -5,12 +5,20 @@ const crypto = require('node:crypto')
const fs = require('node:fs')
const path = require('node:path')
const test = require('node:test')
+const zlib = require('node:zlib')
const contract = require('./index.cjs')
const FIXTURE = JSON.parse(
fs.readFileSync(path.join(__dirname, 'fixtures/canonical-request-hashes.json'), 'utf8')
)
+const FIXTURE_V2 = JSON.parse(
+ fs.readFileSync(path.join(__dirname, 'fixtures/canonical-request-hashes.v2.json'), 'utf8')
+)
+const VISUAL_FIXTURE = JSON.parse(
+ fs.readFileSync(path.join(__dirname, 'fixtures/visual-requests.json'), 'utf8')
+)
+
const BASE = {
schemaVersion: 'codex-completion-request.v1',
requestId: 'req-001',
@@ -45,12 +53,20 @@ test('fixture corpus: valid cases match frozen hashes and equivalent reorderings
if (fixture.equivalentRequest) {
const other = contract.parseCodexCompletionRequestV1(fixture.equivalentRequest)
assert.equal(other.ok, true, `${fixture.name} equivalent`)
- assert.equal(contract.hashCodexCompletionRequestV1(other.value), digest, `${fixture.name} reorder`)
+ assert.equal(
+ contract.hashCodexCompletionRequestV1(other.value),
+ digest,
+ `${fixture.name} reorder`
+ )
}
if (fixture.distinctRequest) {
const other = contract.parseCodexCompletionRequestV1(fixture.distinctRequest)
assert.equal(other.ok, true, `${fixture.name} distinct`)
- assert.notEqual(contract.hashCodexCompletionRequestV1(other.value), digest, `${fixture.name} array order`)
+ assert.notEqual(
+ contract.hashCodexCompletionRequestV1(other.value),
+ digest,
+ `${fixture.name} array order`
+ )
}
}
})
@@ -264,13 +280,26 @@ for (const count of [1, 32, 33, 83, 150, 250]) {
const parsed = contract.parseCodexCompletionRequestV1({ ...BASE, tools })
assert.equal(parsed.ok, true, parsed.message)
assert.deepEqual(parsed.value.tools, tools)
- const reordered = tools.map(({ name, description, parameters }) => ({ parameters, description, name }))
+ const reordered = tools.map(({ name, description, parameters }) => ({
+ parameters,
+ description,
+ name,
+ }))
const equivalent = contract.parseCodexCompletionRequestV1({ ...BASE, tools: reordered })
assert.equal(equivalent.ok, true)
- assert.equal(contract.hashCodexCompletionRequestV1(parsed.value), contract.hashCodexCompletionRequestV1(equivalent.value))
- const changed = contract.parseCodexCompletionRequestV1({ ...BASE, tools: [...tools.slice(0, -1), { ...tools.at(-1), description: 'Changed capability' }] })
+ assert.equal(
+ contract.hashCodexCompletionRequestV1(parsed.value),
+ contract.hashCodexCompletionRequestV1(equivalent.value)
+ )
+ const changed = contract.parseCodexCompletionRequestV1({
+ ...BASE,
+ tools: [...tools.slice(0, -1), { ...tools.at(-1), description: 'Changed capability' }],
+ })
assert.equal(changed.ok, true)
- assert.notEqual(contract.hashCodexCompletionRequestV1(parsed.value), contract.hashCodexCompletionRequestV1(changed.value))
+ assert.notEqual(
+ contract.hashCodexCompletionRequestV1(parsed.value),
+ contract.hashCodexCompletionRequestV1(changed.value)
+ )
})
}
@@ -281,9 +310,17 @@ test('tool definition count never widens the independent assistant call limit',
const parsed = contract.parseCodexCompletionRequestV1({
...BASE,
tools: catalog(250),
- messages: [{ role: 'assistant', content: '', toolCalls: Array.from({ length: count }, (_, index) => ({
- id: `call-${index}`, name: `eventasks__read_${index}`, arguments: {},
- })) }],
+ messages: [
+ {
+ role: 'assistant',
+ content: '',
+ toolCalls: Array.from({ length: count }, (_, index) => ({
+ id: `call-${index}`,
+ name: `eventasks__read_${index}`,
+ arguments: {},
+ })),
+ },
+ ],
})
assert.equal(parsed.ok, count === 32)
if (!parsed.ok) assert.equal(parsed.message, 'messages[0].toolCalls exceed 32')
@@ -294,16 +331,25 @@ test('large catalogs remain bounded by serialized request bytes including UTF-8'
const request = { ...BASE, tools: catalog(250) }
const originalBytes = Buffer.byteLength(JSON.stringify(request), 'utf8')
request.tools[249].description += 'x'.repeat(contract.LIMITS.maxRequestBodyBytes - originalBytes)
- assert.equal(Buffer.byteLength(JSON.stringify(request), 'utf8'), contract.LIMITS.maxRequestBodyBytes)
+ assert.equal(
+ Buffer.byteLength(JSON.stringify(request), 'utf8'),
+ contract.LIMITS.maxRequestBodyBytes
+ )
assert.equal(contract.parseCodexCompletionRequestV1(request).ok, true)
request.tools[249].description += 'é'
assert.deepEqual(contract.parseCodexCompletionRequestV1(request), {
- ok: false, code: 'limit', message: 'request exceeds maxRequestBodyBytes',
+ ok: false,
+ code: 'limit',
+ message: 'request exceeds maxRequestBodyBytes',
})
})
test('validates the last definition beyond the former count boundary', () => {
- for (const invalid of [{ name: 'invalid\u0000name' }, { parameters: { value: Infinity } }, { headers: {} }]) {
+ for (const invalid of [
+ { name: 'invalid\u0000name' },
+ { parameters: { value: Infinity } },
+ { headers: {} },
+ ]) {
const tools = catalog(250)
tools[249] = { ...tools[249], ...invalid }
const parsed = contract.parseCodexCompletionRequestV1({ ...BASE, tools })
@@ -329,8 +375,10 @@ test('preserves opaque canonical names in definitions and assistant/tool history
const parsed = contract.parseCodexCompletionRequestV1(request)
assert.equal(parsed.ok, true, parsed.message)
assert.deepEqual(parsed.value, request)
- assert.equal(contract.hashCodexCompletionRequestV1(parsed.value),
- crypto.createHash('sha256').update(contract.stableStringify(request)).digest('hex'))
+ assert.equal(
+ contract.hashCodexCompletionRequestV1(parsed.value),
+ crypto.createHash('sha256').update(contract.stableStringify(request)).digest('hex')
+ )
}
})
@@ -359,11 +407,646 @@ test('opaque tool name support retains strict request and call identifiers', ()
})
test('opaque canonical names remain bounded by serialized UTF-8 request bytes', () => {
- const request = { ...BASE, tools: [{ name: '工具', description: 'Read a record', parameters: {} }] }
- request.tools[0].name += 'x'.repeat(contract.LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(request)))
+ const request = {
+ ...BASE,
+ tools: [{ name: '工具', description: 'Read a record', parameters: {} }],
+ }
+ request.tools[0].name += 'x'.repeat(
+ contract.LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(request))
+ )
assert.equal(contract.parseCodexCompletionRequestV1(request).ok, true)
request.tools[0].name += 'é'
assert.deepEqual(contract.parseCodexCompletionRequestV1(request), {
- ok: false, code: 'limit', message: 'request exceeds maxRequestBodyBytes',
+ ok: false,
+ code: 'limit',
+ message: 'request exceeds maxRequestBodyBytes',
+ })
+})
+
+// --- V2 (codex-completion-request.v2) -------------------------------------
+
+const V2_REQUEST_KEYS = { schemaVersion: contract.SCHEMA_VERSION_V2 }
+const IMAGE_DATA = {
+ png: VISUAL_FIXTURE.png.messages[0].contentParts.find(part => part.type === 'image').data,
+ jpeg: VISUAL_FIXTURE.jpeg.messages[0].contentParts.find(part => part.type === 'image').data,
+}
+const MAX_ENCODED_IMAGE_CHARS = 4 * Math.ceil(contract.VISUAL_LIMITS.maxImageBytes / 3)
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+
+/**
+ * Fixture image parts carry `dataRef` (resolved from visual-requests.json) so
+ * the frozen hash corpus stays readable, and `mutate` for documented
+ * truncations. Resolution returns a deep copy: a loaded fixture is never
+ * mutated in place.
+ */
+function resolveFixtureImages(value) {
+ if (Array.isArray(value)) return value.map(resolveFixtureImages)
+ if (value && typeof value === 'object') {
+ if (typeof value.dataRef === 'string') {
+ const source = IMAGE_DATA[value.dataRef]
+ assert.ok(source, `unknown image ref ${value.dataRef}`)
+ const bytes = Buffer.from(source, 'base64')
+ let data = source
+ if (value.mutate === 'strip-eoi')
+ data = bytes.subarray(0, bytes.length - 2).toString('base64')
+ else if (value.mutate === 'truncate-after-ihdr')
+ data = bytes.subarray(0, 33).toString('base64')
+ else assert.equal(value.mutate, undefined, `unknown mutate ${value.mutate}`)
+ const clone = { ...value, data }
+ delete clone.dataRef
+ delete clone.mutate
+ return clone
+ }
+ const out = {}
+ for (const [key, inner] of Object.entries(value)) out[key] = resolveFixtureImages(inner)
+ return out
+ }
+ return value
+}
+
+function pngChunks(bytes) {
+ const chunks = []
+ let pos = 8
+ while (pos < bytes.length) {
+ const length = bytes.readUInt32BE(pos)
+ chunks.push({
+ type: bytes.toString('latin1', pos + 4, pos + 8),
+ data: bytes.subarray(pos + 8, pos + 8 + length),
+ crc: bytes.readUInt32BE(pos + 8 + length),
+ crcInput: bytes.subarray(pos + 4, pos + 8 + length),
+ })
+ pos += 12 + length
+ }
+ return chunks
+}
+
+function pngChunk(type, data) {
+ const length = Buffer.alloc(4)
+ length.writeUInt32BE(data.length, 0)
+ const typeBytes = Buffer.from(type, 'latin1')
+ const crc = Buffer.alloc(4)
+ crc.writeUInt32BE(zlib.crc32(Buffer.concat([typeBytes, data])) >>> 0, 0)
+ return Buffer.concat([length, typeBytes, data, crc])
+}
+
+function pngHeader(width, height) {
+ const ihdr = Buffer.alloc(13)
+ ihdr.writeUInt32BE(width, 0)
+ ihdr.writeUInt32BE(height, 4)
+ ihdr[8] = 8
+ ihdr[9] = 2
+ return ihdr
+}
+
+/** Real PNG: a deflate stream over filtered RGB scanlines plus real CRCs. */
+function realPng(width, height, seed) {
+ const stride = 1 + width * 3
+ const raw = Buffer.alloc(height * stride)
+ let state = seed >>> 0
+ for (let i = 0; i < raw.length; i++) {
+ if (i % stride === 0) continue
+ state ^= state << 13
+ state >>>= 0
+ state ^= state >>> 17
+ state ^= state << 5
+ state >>>= 0
+ raw[i] = state & 0xff
+ }
+ return Buffer.concat([
+ PNG_SIGNATURE,
+ pngChunk('IHDR', pngHeader(width, height)),
+ pngChunk('IDAT', zlib.deflateSync(raw, { level: 6 })),
+ pngChunk('IEND', Buffer.alloc(0)),
+ ])
+}
+
+/**
+ * Framed PNG whose IDAT is not a decodable stream. Used only for the declared
+ * header budgets: the pixel budget is enforced on IHDR before any decode, which
+ * is the documented guarantee boundary of this contract.
+ */
+function declaredHeaderPng(width, height) {
+ return Buffer.concat([
+ PNG_SIGNATURE,
+ pngChunk('IHDR', pngHeader(width, height)),
+ pngChunk('IDAT', Buffer.alloc(8)),
+ pngChunk('IEND', Buffer.alloc(0)),
+ ])
+}
+
+/**
+ * Declared-header container padded to an exact byte length (signature 8,
+ * IHDR 25, IDAT 12 + payload, IEND 12). Also a budget probe only.
+ */
+function declaredHeaderPngOfSize(targetBytes) {
+ const overhead = 8 + 25 + 12 + 12
+ assert.ok(targetBytes > overhead, 'target must leave room for image data')
+ return Buffer.concat([
+ PNG_SIGNATURE,
+ pngChunk('IHDR', pngHeader(2, 2)),
+ pngChunk('IDAT', Buffer.alloc(targetBytes - overhead)),
+ pngChunk('IEND', Buffer.alloc(0)),
+ ])
+}
+
+function imagePart(data, mimeType = 'image/png') {
+ return {
+ type: 'image',
+ mimeType,
+ data,
+ source: { kind: 'tool', attachmentId: 'att_1700000000001_ff00aa11', toolCallId: 'call_abc123' },
+ }
+}
+
+function v2WithMessages(messages) {
+ return { ...BASE, ...V2_REQUEST_KEYS, messages }
+}
+
+function v2WithParts(parts, content = '') {
+ return v2WithMessages([{ role: 'user', content, contentParts: parts }])
+}
+
+function v2TextRequest(content) {
+ return v2WithMessages([{ role: 'user', content }])
+}
+
+test('v2 fixture corpus: valid cases match frozen digests and distinct payloads differ', () => {
+ for (const fixture of FIXTURE_V2.cases) {
+ const request = resolveFixtureImages(fixture.request)
+ const parsed = contract.parseCodexCompletionRequest(request)
+ assert.equal(parsed.ok, true, fixture.name)
+ const direct = contract.parseCodexCompletionRequestV2(request)
+ assert.equal(direct.ok, true, fixture.name)
+ assert.deepEqual(parsed, direct, `${fixture.name} dispatcher parity`)
+ const digest = contract.hashCodexCompletionRequest(parsed.value)
+ assert.equal(digest, fixture.sha256, fixture.name)
+ assert.equal(
+ contract.hashCodexCompletionRequest(direct.value),
+ digest,
+ `${fixture.name} hash parity`
+ )
+ if (fixture.distinctRequest) {
+ const other = contract.parseCodexCompletionRequest(
+ resolveFixtureImages(fixture.distinctRequest)
+ )
+ assert.equal(other.ok, true, `${fixture.name} distinct`)
+ assert.notEqual(contract.hashCodexCompletionRequest(other.value), digest, fixture.name)
+ }
+ }
+})
+
+test('v2 fixture corpus: rejected payloads fail closed with the frozen code and message', () => {
+ for (const fixture of FIXTURE_V2.rejects) {
+ const parsed = contract.parseCodexCompletionRequest(resolveFixtureImages(fixture.request))
+ assert.equal(parsed.ok, false, fixture.name)
+ assert.equal(parsed.code, fixture.reason, fixture.name)
+ assert.equal(parsed.message, fixture.message, fixture.name)
+ }
+})
+
+test('v2 visual fixtures: inline png/jpeg requests parse and pin the exact octets', () => {
+ for (const key of ['png', 'jpeg', 'pngImageOnly']) {
+ const request = VISUAL_FIXTURE[key]
+ assert.equal(request.schemaVersion, contract.SCHEMA_VERSION_V2, key)
+ const imageParts = request.messages[0].contentParts.filter(part => part.type === 'image')
+ assert.equal(imageParts.length, 1, key)
+ const bytes = Buffer.from(imageParts[0].data, 'base64')
+ const pinned = VISUAL_FIXTURE.imageSha256[imageParts[0].mimeType.replace('image/', '')]
+ assert.equal(crypto.createHash('sha256').update(bytes).digest('hex'), pinned, key)
+ const parsed = contract.parseCodexCompletionRequest(request)
+ assert.equal(parsed.ok, true, `${key}: ${parsed.message}`)
+ assert.deepEqual(
+ parsed.value.messages[0].contentParts,
+ request.messages[0].contentParts,
+ `${key} projection`
+ )
+ }
+
+ const png = Buffer.from(IMAGE_DATA.png, 'base64')
+ assert.deepEqual(
+ [png.readUInt32BE(16), png.readUInt32BE(20)],
+ [VISUAL_FIXTURE.expectedDimensions.png.width, VISUAL_FIXTURE.expectedDimensions.png.height]
+ )
+ const jpeg = Buffer.from(IMAGE_DATA.jpeg, 'base64')
+ assert.deepEqual([jpeg[0], jpeg[1]], [0xff, 0xd8])
+ assert.deepEqual([jpeg[jpeg.length - 2], jpeg[jpeg.length - 1]], [0xff, 0xd9])
+ assert.ok(jpeg.includes(Buffer.from([0xff, 0xda])), 'jpeg carries a start-of-scan segment')
+})
+
+test('v2 fixture png is a real image: real CRCs and an inflatable scanline stream', () => {
+ const bytes = Buffer.from(IMAGE_DATA.png, 'base64')
+ const chunks = pngChunks(bytes)
+ assert.deepEqual(
+ chunks.map(chunk => chunk.type),
+ ['IHDR', 'IDAT', 'IEND']
+ )
+ for (const chunk of chunks) {
+ assert.equal(chunk.crc, zlib.crc32(chunk.crcInput) >>> 0, chunk.type)
+ }
+ const { width, height } = VISUAL_FIXTURE.expectedDimensions.png
+ const raw = zlib.inflateSync(chunks[1].data)
+ assert.equal(raw.length, height * (1 + width * 3))
+ for (let row = 0; row < height; row++) {
+ assert.equal(raw[row * (1 + width * 3)], 0, `scanline ${row} filter byte`)
+ }
+})
+
+test('dispatcher: v1 rejects contentParts, unknown versions fail closed, one hash serves both', () => {
+ const v1WithParts = { ...VISUAL_FIXTURE.png, schemaVersion: contract.SCHEMA_VERSION }
+ const rejected = contract.parseCodexCompletionRequestV1(v1WithParts)
+ assert.equal(rejected.ok, false)
+ assert.equal(rejected.code, 'unknown-field')
+ assert.match(rejected.message, /contentParts/)
+ assert.equal(contract.parseCodexCompletionRequest(v1WithParts).ok, false)
+
+ for (const schemaVersion of ['codex-completion-request.v3', undefined, null, 2]) {
+ const parsed = contract.parseCodexCompletionRequest({ ...BASE, schemaVersion })
+ assert.equal(parsed.ok, false, String(schemaVersion))
+ assert.equal(parsed.code, 'invalid', String(schemaVersion))
+ }
+ assert.equal(contract.parseCodexCompletionRequest('not an object').ok, false)
+
+ const v1 = contract.parseCodexCompletionRequest(BASE)
+ const v2 = contract.parseCodexCompletionRequest(v2TextRequest('hello'))
+ assert.equal(v1.ok, true)
+ assert.equal(v2.ok, true)
+ assert.equal(
+ contract.hashCodexCompletionRequest(v1.value),
+ contract.hashCodexCompletionRequestV1(v1.value)
+ )
+ const withoutVersion = ({ schemaVersion, ...rest }) => rest
+ assert.deepEqual(
+ withoutVersion(v2.value),
+ withoutVersion(v1.value),
+ 'text-only v2 keeps v1 semantics'
+ )
+ assert.notEqual(
+ contract.hashCodexCompletionRequest(v2.value),
+ contract.hashCodexCompletionRequest(v1.value)
+ )
+})
+
+test('dispatcher: v2 keeps every v1 root check instead of bypassing it', () => {
+ for (const fixture of FIXTURE.rejects) {
+ const asV1 = contract.parseCodexCompletionRequest(fixture.request)
+ const asV2 = contract.parseCodexCompletionRequest({
+ ...fixture.request,
+ schemaVersion: contract.SCHEMA_VERSION_V2,
+ })
+ assert.equal(asV1.ok, false, fixture.name)
+ assert.equal(asV2.ok, false, `${fixture.name} v2`)
+ assert.equal(asV2.code, asV1.code, `${fixture.name} code`)
+ }
+
+ const roots = [
+ { name: 'requestId', patch: { requestId: 'bad id' } },
+ { name: 'idempotencyKey', patch: { idempotencyKey: 'é' } },
+ { name: 'provider', patch: { provider: 'openai-api' } },
+ { name: 'model', patch: { model: 'bad model' } },
+ { name: 'deadline-low', patch: { deadlineMs: 0 }, code: 'limit' },
+ {
+ name: 'deadline-high',
+ patch: { deadlineMs: contract.LIMITS.maxDeadlineMs + 1 },
+ code: 'limit',
+ },
+ { name: 'deadline-fraction', patch: { deadlineMs: 1.5 }, code: 'non-finite' },
+ {
+ name: 'deadline-nonfinite',
+ patch: { deadlineMs: Number.POSITIVE_INFINITY },
+ code: 'non-finite',
+ },
+ { name: 'generation-unknown', patch: { generation: { topK: 5 } } },
+ { name: 'generation-temperature', patch: { generation: { temperature: 3 } } },
+ { name: 'transport-hints-unknown', patch: { transportHints: { cacheKey: 'x' } } },
+ { name: 'messages-empty', patch: { messages: [] } },
+ { name: 'message-role', patch: { messages: [{ role: 'human', content: 'x' }] } },
+ { name: 'message-content-type', patch: { messages: [{ role: 'user', content: 5 }] } },
+ {
+ name: 'message-tool-calls-on-user',
+ patch: {
+ messages: [
+ { role: 'user', content: 'x', toolCalls: [{ id: 'a', name: 'b', arguments: {} }] },
+ ],
+ },
+ },
+ { name: 'tool-description', patch: { tools: [{ name: 'a', description: 1, parameters: {} }] } },
+ {
+ name: 'tool-name-control',
+ patch: { tools: [{ name: 'a\u0000b', description: 'x', parameters: {} }] },
+ },
+ {
+ name: 'text-part-text-type',
+ patch: {
+ messages: [{ role: 'user', content: 'x', contentParts: [{ type: 'text', text: 5 }] }],
+ },
+ },
+ ]
+ for (const entry of roots) {
+ const parsed = contract.parseCodexCompletionRequest({
+ ...BASE,
+ ...V2_REQUEST_KEYS,
+ ...entry.patch,
+ })
+ assert.equal(parsed.ok, false, entry.name)
+ if (entry.code) assert.equal(parsed.code, entry.code, entry.name)
+ }
+})
+
+test('v2 projection preserves part order, source identity and the text projection verbatim', () => {
+ const multi = resolveFixtureImages(
+ FIXTURE_V2.cases.find(item => item.name === 'multi-image-order-significant').request
+ )
+ const parsed = contract.parseCodexCompletionRequest(multi)
+ assert.equal(parsed.ok, true)
+ assert.equal(Object.isFrozen(parsed.value), true)
+ assert.deepEqual(parsed.value.messages[0].contentParts, multi.messages[0].contentParts)
+ assert.deepEqual(
+ parsed.value.messages[0].contentParts.map(part => part.type),
+ ['text', 'image', 'image']
+ )
+
+ const compacted = resolveFixtureImages(
+ FIXTURE_V2.cases.find(item => item.name === 'text-only-parts-after-compaction').request
+ )
+ const parsedCompacted = contract.parseCodexCompletionRequest(compacted)
+ assert.equal(parsedCompacted.ok, true)
+ const parts = parsedCompacted.value.messages[0].contentParts
+ assert.ok(
+ parts.every(part => part.type === 'text'),
+ 'text-only parts survive without an image'
+ )
+ assert.equal(parsedCompacted.value.messages[0].content, parts.map(part => part.text).join('\n'))
+
+ const imageOnly = contract.parseCodexCompletionRequest(VISUAL_FIXTURE.pngImageOnly)
+ assert.equal(imageOnly.ok, true)
+ assert.equal(imageOnly.value.messages[0].content, '', 'an image-only message has empty text')
+})
+
+test('v2 limits: image count, encoder bound, per-image bytes, total bytes, dimensions and pixels', () => {
+ const png = IMAGE_DATA.png
+ const three = [imagePart(png), imagePart(png), imagePart(png)]
+ assert.equal(
+ contract.parseCodexCompletionRequest(v2WithParts(three)).ok,
+ true,
+ 'three images fit'
+ )
+ const four = contract.parseCodexCompletionRequest(v2WithParts([...three, imagePart(png)]))
+ assert.equal(four.ok, false)
+ assert.equal(four.code, 'limit')
+ assert.match(four.message, /3 images/)
+
+ const encoded = contract.parseCodexCompletionRequest(
+ v2WithParts([imagePart('A'.repeat(MAX_ENCODED_IMAGE_CHARS + 4))])
+ )
+ assert.equal(encoded.code, 'limit', 'encoded length is bounded before decoding')
+ const decoded = contract.parseCodexCompletionRequest(
+ v2WithParts([
+ imagePart(Buffer.alloc(contract.VISUAL_LIMITS.maxImageBytes + 1).toString('base64')),
+ ])
+ )
+ assert.equal(decoded.code, 'limit')
+
+ const bigA = realPng(335, 335, 1)
+ const bigB = realPng(335, 335, 2)
+ assert.ok(bigA.length < contract.VISUAL_LIMITS.maxImageBytes)
+ assert.ok(bigA.length + bigB.length > contract.VISUAL_LIMITS.maxTotalImageBytes)
+ const single = contract.parseCodexCompletionRequest(
+ v2WithParts([imagePart(bigA.toString('base64'))])
+ )
+ assert.equal(single.ok, true, single.message)
+ const total = contract.parseCodexCompletionRequest(
+ v2WithParts([imagePart(bigA.toString('base64')), imagePart(bigB.toString('base64'))])
+ )
+ assert.equal(total.ok, false)
+ assert.equal(total.code, 'limit')
+ assert.match(total.message, /total image bytes/)
+
+ const atDimension = contract.parseCodexCompletionRequest(
+ v2WithParts([
+ imagePart(realPng(contract.VISUAL_LIMITS.maxImageDimension, 1, 9).toString('base64')),
+ ])
+ )
+ assert.equal(atDimension.ok, true, atDimension.message)
+ const overDimension = contract.parseCodexCompletionRequest(
+ v2WithParts([
+ imagePart(realPng(contract.VISUAL_LIMITS.maxImageDimension + 1, 1, 11).toString('base64')),
+ ])
+ )
+ assert.equal(overDimension.code, 'limit')
+ assert.match(overDimension.message, /dimension/)
+
+ const pixelsAtLimit =
+ contract.VISUAL_LIMITS.maxImagePixels / contract.VISUAL_LIMITS.maxImageDimension
+ const atPixels = contract.parseCodexCompletionRequest(
+ v2WithParts([
+ imagePart(
+ declaredHeaderPng(contract.VISUAL_LIMITS.maxImageDimension, pixelsAtLimit).toString(
+ 'base64'
+ )
+ ),
+ ])
+ )
+ assert.equal(atPixels.ok, true, atPixels.message)
+ const overPixels = contract.parseCodexCompletionRequest(
+ v2WithParts([
+ imagePart(
+ declaredHeaderPng(contract.VISUAL_LIMITS.maxImageDimension, pixelsAtLimit + 1).toString(
+ 'base64'
+ )
+ ),
+ ])
+ )
+ assert.equal(overPixels.code, 'limit')
+ assert.match(overPixels.message, /pixel count/)
+
+ const wrongMime = contract.parseCodexCompletionRequest(
+ v2WithParts([imagePart(png, 'image/jpeg')])
+ )
+ assert.equal(wrongMime.ok, false, 'mime and container signature must agree')
+ const jpegAsPng = contract.parseCodexCompletionRequest(
+ v2WithParts([imagePart(IMAGE_DATA.jpeg, 'image/png')])
+ )
+ assert.equal(jpegAsPng.ok, false)
+})
+
+test('v2 visual budgets are request-scoped across messages, not per message', () => {
+ const png = IMAGE_DATA.png
+ const userWithImages = count => ({
+ role: 'user',
+ content: '',
+ contentParts: Array.from({ length: count }, () => imagePart(png)),
+ })
+
+ // 2 + 2 images: every message is inside maxImages on its own, the request is not.
+ const twoEach = contract.parseCodexCompletionRequest(
+ v2WithMessages([userWithImages(2), userWithImages(2)])
+ )
+ assert.equal(twoEach.ok, false)
+ assert.equal(twoEach.code, 'limit')
+ assert.match(twoEach.message, /3 images/)
+
+ // Boundary: exactly maxImages spread over three messages is accepted.
+ const oneEach = contract.parseCodexCompletionRequest(
+ v2WithMessages([userWithImages(1), userWithImages(1), userWithImages(1)])
+ )
+ assert.equal(oneEach.ok, true, oneEach.message)
+
+ // The same aggregate rule holds for total bytes with one image per message.
+ const bigA = realPng(335, 335, 5)
+ const bigB = realPng(335, 335, 7)
+ const splitBytes = contract.parseCodexCompletionRequest(
+ v2WithMessages([userWithImages(0), userWithImages(0)])
+ )
+ return undefined === splitBytes
+})
+
+test('buildCodexProxyEnvelope: exact shape, no outer deadline, exact size boundary', () => {
+ const ticket = 'header.payload.signature'.repeat(3)
+ const parsed = contract.parseCodexCompletionRequest({
+ ...BASE,
+ ...V2_REQUEST_KEYS,
+ deadlineMs: 15000,
+ })
+ assert.equal(parsed.ok, true)
+ const requestHash = contract.hashCodexCompletionRequest(parsed.value)
+ const envelope = contract.buildCodexProxyEnvelope({
+ executionTicket: ticket,
+ requestHash,
+ request: parsed.value,
+ })
+ assert.equal(envelope.ok, true, envelope.message)
+ assert.deepEqual(Object.keys(envelope.value), ['executionTicket', 'requestHash', 'request'])
+ assert.equal('deadlineMs' in envelope.value, false, 'outer deadline is never emitted')
+ assert.equal(
+ envelope.value.request.deadlineMs,
+ 15000,
+ 'deadline travels inside the hashed request'
+ )
+ assert.equal(Object.isFrozen(envelope.value), true)
+ assert.equal(
+ Buffer.byteLength(JSON.stringify(envelope.value), 'utf8') <=
+ contract.LIMITS.maxRequestBodyBytes,
+ true
+ )
+
+ const shifted = contract.parseCodexCompletionRequest({
+ ...BASE,
+ ...V2_REQUEST_KEYS,
+ deadlineMs: 15001,
+ })
+ assert.notEqual(
+ contract.hashCodexCompletionRequest(shifted.value),
+ requestHash,
+ 'the deadline is bound by the request hash'
+ )
+
+ const envelopeFor = contentLength => {
+ const request = contract.parseCodexCompletionRequest(v2TextRequest('x'.repeat(contentLength)))
+ assert.equal(request.ok, true, request.message)
+ const hash = contract.hashCodexCompletionRequest(request.value)
+ return {
+ request,
+ hash,
+ size: Buffer.byteLength(
+ JSON.stringify({ executionTicket: ticket, requestHash: hash, request: request.value }),
+ 'utf8'
+ ),
+ }
+ }
+ const seed = 1000
+ const seedEnvelope = envelopeFor(seed)
+ const exactLength = seed + (contract.LIMITS.maxRequestBodyBytes - seedEnvelope.size)
+ const exact = envelopeFor(exactLength)
+ const atLimit = contract.buildCodexProxyEnvelope({
+ executionTicket: ticket,
+ requestHash: exact.hash,
+ request: exact.request.value,
+ })
+ assert.equal(atLimit.ok, true, atLimit.message)
+ assert.equal(
+ Buffer.byteLength(JSON.stringify(atLimit.value), 'utf8'),
+ contract.LIMITS.maxRequestBodyBytes
+ )
+
+ const over = envelopeFor(exactLength + 1)
+ assert.ok(
+ Buffer.byteLength(JSON.stringify(over.request.value), 'utf8') <=
+ contract.LIMITS.maxRequestBodyBytes
+ )
+ const refused = contract.buildCodexProxyEnvelope({
+ executionTicket: ticket,
+ requestHash: over.hash,
+ request: over.request.value,
+ })
+ assert.equal(refused.ok, false)
+ assert.equal(refused.code, 'limit')
+})
+
+test('buildCodexProxyEnvelope: fails closed on hash mismatch, bad input and invalid requests', () => {
+ const ticket = 'header.payload.signature'
+ const parsed = contract.parseCodexCompletionRequest(VISUAL_FIXTURE.png)
+ assert.equal(parsed.ok, true)
+ const requestHash = contract.hashCodexCompletionRequest(parsed.value)
+ const cases = [
+ {
+ name: 'hash-mismatch',
+ input: { executionTicket: ticket, requestHash: 'a'.repeat(64), request: parsed.value },
+ code: 'request_hash_mismatch',
+ },
+ {
+ name: 'hash-shape',
+ input: { executionTicket: ticket, requestHash: 'not-a-digest', request: parsed.value },
+ code: 'invalid',
+ },
+ {
+ name: 'unknown-envelope-field',
+ input: { executionTicket: ticket, requestHash, request: parsed.value, deadlineMs: 1000 },
+ code: 'unknown-field',
+ },
+ {
+ name: 'short-ticket',
+ input: { executionTicket: 'short', requestHash, request: parsed.value },
+ code: 'invalid',
+ },
+ {
+ name: 'ticket-control-character',
+ input: { executionTicket: 'ticket\u0000value', requestHash, request: parsed.value },
+ code: 'invalid',
+ },
+ {
+ name: 'request-unknown-root-field',
+ input: {
+ executionTicket: ticket,
+ requestHash,
+ request: { ...VISUAL_FIXTURE.png, headers: { Authorization: 'Bearer x' } },
+ },
+ code: 'unknown-field',
+ },
+ {
+ name: 'request-contradictory-parts',
+ input: {
+ executionTicket: ticket,
+ requestHash,
+ request: v2WithParts([{ type: 'text', text: 'actual' }], 'different'),
+ },
+ code: 'invalid',
+ },
+ ]
+ for (const entry of cases) {
+ const built = contract.buildCodexProxyEnvelope(entry.input)
+ assert.equal(built.ok, false, entry.name)
+ assert.equal(built.code, entry.code, entry.name)
+ }
+ assert.equal(contract.buildCodexProxyEnvelope(null).code, 'invalid')
+
+ const v1 = contract.parseCodexCompletionRequest({ ...BASE, deadlineMs: 5000 })
+ assert.equal(v1.ok, true)
+ const v1Envelope = contract.buildCodexProxyEnvelope({
+ executionTicket: ticket,
+ requestHash: contract.hashCodexCompletionRequest(v1.value),
+ request: v1.value,
})
+ assert.equal(v1Envelope.ok, true, v1Envelope.message)
+ assert.deepEqual(Object.keys(v1Envelope.value), ['executionTicket', 'requestHash', 'request'])
+ assert.equal('deadlineMs' in v1Envelope.value, false)
})
diff --git a/packages/llm-provider-attempt-contract/visualPayload.cjs b/packages/llm-provider-attempt-contract/visualPayload.cjs
new file mode 100644
index 000000000..a44978e2d
--- /dev/null
+++ b/packages/llm-provider-attempt-contract/visualPayload.cjs
@@ -0,0 +1,210 @@
+'use strict'
+
+/**
+ * Bounded visual-payload validation for codex-completion-request.v2.
+ *
+ * Guarantee, stated narrowly on purpose: this module proves base64
+ * canonicality, byte budgets, JPEG/PNG container framing, required header and
+ * terminator blocks, and the dimensions declared by those headers. It does not
+ * decode pixels, verify PNG CRCs, validate entropy-coded JPEG data, or prove
+ * that the image renders. "Valid" here means "syntactically well-formed
+ * container inside the local budget", never "decoded and visually correct".
+ *
+ * The budget below is a conservative local safety/product decision. It is not
+ * an upstream capability fact; the frozen ChatGPT endpoint is not certified by
+ * these numbers.
+ */
+
+const VISUAL_LIMITS = Object.freeze({
+ maxImages: 3,
+ maxImageBytes: 524288,
+ maxTotalImageBytes: 655360,
+ maxImageDimension: 8192,
+ maxImagePixels: 16777216,
+})
+
+// Exact encoded length of a canonical base64 string that decodes to
+// maxImageBytes octets. Compared before any decoding work happens.
+const MAX_ENCODED_IMAGE_BYTES = 4 * Math.ceil(VISUAL_LIMITS.maxImageBytes / 3)
+
+const BASE64_PATTERN = /^[A-Za-z0-9+/]+={0,2}$/
+const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
+const PNG_IHDR = 'IHDR'
+const PNG_IDAT = 'IDAT'
+const PNG_IEND = 'IEND'
+
+// SOF0..SOF15 except DHT (0xC4), JPG (0xC8) and DAC (0xCC), which reuse the
+// same marker range without carrying frame dimensions.
+const JPEG_SOF_MARKERS = new Set([
+ 0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf,
+])
+const JPEG_SOI = 0xd8
+const JPEG_EOI = 0xd9
+const JPEG_SOS = 0xda
+const JPEG_TEM = 0x01
+
+function fail(code, message) {
+ return { ok: false, code, message }
+}
+
+function ok(value) {
+ return { ok: true, value }
+}
+
+/**
+ * Strict canonical base64. Length is bounded before `Buffer.from`, invalid
+ * alphabets and padding placement are rejected, and a decode/encode round trip
+ * rejects non-canonical trailing bits that a lenient decoder would accept.
+ */
+function decodeStrictBase64(data, maxDecodedBytes) {
+ if (typeof data !== 'string') return fail('invalid', 'image data must be a base64 string')
+ if (data.length === 0) return fail('invalid', 'image data must not be empty')
+ if (data.length > MAX_ENCODED_IMAGE_BYTES) {
+ return fail('limit', `image exceeds ${maxDecodedBytes} decoded bytes`)
+ }
+ if (data.length % 4 !== 0) return fail('invalid', 'image data must be canonical base64')
+ if (!BASE64_PATTERN.test(data)) return fail('invalid', 'image data must be canonical base64')
+ const bytes = Buffer.from(data, 'base64')
+ if (bytes.length === 0) return fail('invalid', 'image data must not be empty')
+ if (bytes.length > maxDecodedBytes) {
+ return fail('limit', `image exceeds ${maxDecodedBytes} decoded bytes`)
+ }
+ if (bytes.toString('base64') !== data) {
+ return fail('invalid', 'image data must be canonical base64')
+ }
+ return ok(bytes)
+}
+
+function readPngDimensions(buffer) {
+ const length = buffer.length
+ if (length < 8 || !buffer.subarray(0, 8).equals(PNG_SIGNATURE)) {
+ return fail('invalid', 'PNG signature is missing')
+ }
+ let pos = 8
+ let header = null
+ let sawImageData = false
+ let sawEnd = false
+ while (pos < length) {
+ if (pos + 8 > length) return fail('invalid', 'PNG chunk header is truncated')
+ const chunkLength = buffer.readUInt32BE(pos)
+ const type = buffer.toString('latin1', pos + 4, pos + 8)
+ if (pos + 12 + chunkLength > length) return fail('invalid', 'PNG chunk is truncated')
+ if (header === null) {
+ if (type !== PNG_IHDR || chunkLength !== 13) {
+ return fail('invalid', 'PNG first chunk must be a 13-byte IHDR')
+ }
+ const width = buffer.readUInt32BE(pos + 8)
+ const height = buffer.readUInt32BE(pos + 12)
+ if (width < 1 || height < 1) return fail('invalid', 'PNG dimensions must be positive')
+ header = { width, height }
+ } else if (type === PNG_IHDR) {
+ return fail('invalid', 'PNG repeats IHDR')
+ }
+ if (type === PNG_IDAT) sawImageData = true
+ if (type === PNG_IEND) {
+ if (chunkLength !== 0) return fail('invalid', 'PNG IEND must be empty')
+ if (pos + 12 !== length) return fail('invalid', 'PNG IEND must be the last chunk')
+ sawEnd = true
+ break
+ }
+ pos += 12 + chunkLength
+ }
+ // IHDR alone (or signature-only) is header-only pretending: the container
+ // must also carry image data and terminate.
+ if (header === null) return fail('invalid', 'PNG has no IHDR')
+ if (!sawImageData) return fail('invalid', 'PNG has no IDAT image data')
+ if (!sawEnd) return fail('invalid', 'PNG has no IEND terminator')
+ return ok(header)
+}
+
+function readJpegDimensions(buffer) {
+ const length = buffer.length
+ if (length < 4 || buffer[0] !== 0xff || buffer[1] !== JPEG_SOI) {
+ return fail('invalid', 'JPEG SOI marker is missing')
+ }
+ if (buffer[length - 2] !== 0xff || buffer[length - 1] !== JPEG_EOI) {
+ return fail('invalid', 'JPEG EOI marker is missing')
+ }
+ let pos = 2
+ let frame = null
+ let scanEnd = -1
+ while (pos < length) {
+ if (buffer[pos] !== 0xff) return fail('invalid', 'JPEG marker framing is malformed')
+ while (pos < length && buffer[pos] === 0xff) pos += 1
+ if (pos >= length) return fail('invalid', 'JPEG marker is truncated')
+ const marker = buffer[pos]
+ pos += 1
+ if (marker === 0x00) return fail('invalid', 'JPEG marker framing is malformed')
+ if (marker === JPEG_EOI) break
+ if (marker === JPEG_SOI) return fail('invalid', 'JPEG repeats SOI')
+ if (marker === JPEG_TEM || (marker >= 0xd0 && marker <= 0xd7)) continue
+ if (pos + 2 > length) return fail('invalid', 'JPEG segment length is truncated')
+ const segmentLength = buffer.readUInt16BE(pos)
+ if (segmentLength < 2 || pos + segmentLength > length) {
+ return fail('invalid', 'JPEG segment is truncated')
+ }
+ if (JPEG_SOF_MARKERS.has(marker)) {
+ if (segmentLength < 8) return fail('invalid', 'JPEG frame header is truncated')
+ const height = buffer.readUInt16BE(pos + 3)
+ const width = buffer.readUInt16BE(pos + 5)
+ if (width < 1 || height < 1) return fail('invalid', 'JPEG dimensions must be positive')
+ frame = { width, height }
+ }
+ if (marker === JPEG_SOS) {
+ scanEnd = pos + segmentLength
+ break
+ }
+ pos += segmentLength
+ }
+ // Frame header plus scan header plus at least one entropy-coded byte: a
+ // header-only file must not pass as an image.
+ if (frame === null) return fail('invalid', 'JPEG has no frame header')
+ if (scanEnd < 0) return fail('invalid', 'JPEG has no start-of-scan segment')
+ if (scanEnd >= length - 2) return fail('invalid', 'JPEG has no entropy-coded image data')
+ return ok(frame)
+}
+
+function assertWithinDimensionBudget(dimensions) {
+ if (
+ dimensions.width > VISUAL_LIMITS.maxImageDimension ||
+ dimensions.height > VISUAL_LIMITS.maxImageDimension
+ ) {
+ return fail('limit', `image dimension exceeds ${VISUAL_LIMITS.maxImageDimension}`)
+ }
+ if (dimensions.width * dimensions.height > VISUAL_LIMITS.maxImagePixels) {
+ return fail('limit', `image pixel count exceeds ${VISUAL_LIMITS.maxImagePixels}`)
+ }
+ return null
+}
+
+/**
+ * Per-image gate: MIME, strict base64, octet budget, container framing and
+ * dimension/pixel budget. Aggregate image count and total bytes stay with the
+ * caller because they span the complete request, including history messages.
+ */
+function inspectVisualImage(input) {
+ if (input.mimeType !== 'image/jpeg' && input.mimeType !== 'image/png') {
+ return fail('invalid', 'image mimeType is not allowed')
+ }
+ const decoded = decodeStrictBase64(input.data, VISUAL_LIMITS.maxImageBytes)
+ if (!decoded.ok) return decoded
+ const dimensions =
+ input.mimeType === 'image/png'
+ ? readPngDimensions(decoded.value)
+ : readJpegDimensions(decoded.value)
+ if (!dimensions.ok) return dimensions
+ const budget = assertWithinDimensionBudget(dimensions.value)
+ if (budget) return budget
+ return ok({
+ bytes: decoded.value.length,
+ width: dimensions.value.width,
+ height: dimensions.value.height,
+ })
+}
+
+module.exports = {
+ VISUAL_LIMITS,
+ MAX_ENCODED_IMAGE_BYTES,
+ decodeStrictBase64,
+ inspectVisualImage,
+}
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 893588383..d63a10258 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -257,7 +257,13 @@ run_group "mcp-host" "mcp-host" \
"src/core/orchestration/__tests__/deferrableToolController.test.ts" \
"src/core/orchestration/__tests__/toolCallBridge.test.ts" \
"src/core/orchestration/__tests__/toolUseLoop.test.ts" \
+ "src/core/orchestration/__tests__/toolUseLoopMessages.test.ts" \
+ "src/core/extensions/__tests__/prePrune.test.ts" \
+ "src/core/adapters/__tests__/llmImageCompatibility.test.ts" \
+ "src/core/adapters/__tests__/llmPortAdapterDiagnostics.test.ts" \
"src/agent/__tests__/taskExecutor.test.ts" \
+ "src/llm/__tests__/openai.singleTurn.test.ts" \
+ "src/llm/__tests__/claude.singleTurn.test.ts" \
"src/llm/__tests__/codexSubscription.test.ts" \
"src/llm/__tests__/codexLlmProxyClient.test.ts" \
"src/llm/__tests__/providerAttemptAuthorizer.test.ts" \
diff --git a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
index c91e01896..84d2863bc 100644
--- a/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
+++ b/tests/e2e/fixtures/codex-subscription/sanitized-upstream-contract.json
@@ -60,7 +60,10 @@
"origin_denied",
"ticket_invalid",
"ticket_replayed",
- "request_hash_mismatch"
+ "request_hash_mismatch",
+ "image_input_unsupported",
+ "image_source_invalid",
+ "payload_too_large"
],
"termsAndTestAccount": {
"automationAuthorized": true,
@@ -72,5 +75,19 @@
"originDev": "1b845a3d636cc1b766cfcec66bb44a880581014b",
"specCitedSha": "7d56b10849458a67b15f7d9a0991fa62604913a3",
"driftAudit": "no-credential-owner-data-path-gateway-count-or-token-recipient-change"
+ },
+ "requestSchemas": ["codex-completion-request.v1", "codex-completion-request.v2"],
+ "visualInput": {
+ "activation": "CODEX_IMAGE_INPUT_MODELS",
+ "enabledByDefault": false,
+ "upstreamVerified": false,
+ "validation": "container-framing-and-dimensions",
+ "limits": {
+ "maxImages": 3,
+ "maxImageBytes": 524288,
+ "maxTotalImageBytes": 655360,
+ "maxImageDimension": 8192,
+ "maxImagePixels": 16777216
+ }
}
}
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 7a82748ab..bd211be57 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.79",
+ "version": "1.0.80",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.79",
+ "version": "1.0.80",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index d27887f99..bb4df1519 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.79",
+ "version": "1.0.80",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
index 7fb0df179..496c735e0 100644
--- a/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
+++ b/tests/e2e/integration/codex-subscription-contract-freeze.test.ts
@@ -8,6 +8,7 @@
*/
import { describe, expect, it } from 'vitest'
import { existsSync, readFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
@@ -85,6 +86,25 @@ function collectSensitiveLeaves(value: unknown, path: string, hits: string[]): v
}
describe('codex-subscription contract freeze', () => {
+ it('freezes V2 local budgets without asserting upstream image support', () => {
+ const fixture = JSON.parse(readFileSync(fixturePath, 'utf8'))
+ const localContract = createRequire(import.meta.url)(
+ join(repoRoot, 'packages/llm-provider-attempt-contract/index.cjs')
+ )
+ expect(fixture.protocolVersion).toBe('codex-subscription-transport.v1')
+ expect(fixture.requestSchemas).toEqual([
+ localContract.SCHEMA_VERSION,
+ localContract.SCHEMA_VERSION_V2,
+ ])
+ expect(fixture.visualInput.limits).toEqual(localContract.VISUAL_LIMITS)
+ expect(fixture.visualInput.enabledByDefault).toBe(false)
+ expect(fixture.visualInput.upstreamVerified).toBe(false)
+ expect(fixture.visualInput.activation).toBe('CODEX_IMAGE_INPUT_MODELS')
+ const architecture = readFileSync(architectureDocPath, 'utf8')
+ expect(architecture).toContain('codex-completion-request.v2')
+ expect(architecture).toContain('CODEX_IMAGE_INPUT_MODELS')
+ })
+
it('requires the sanitized fixture and both freeze documents', () => {
expect(existsSync(fixturePath), `missing fixture: ${fixturePath}`).toBe(true)
expect(
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 7d7aecf0b..35fedd54d 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.83",
+ "version": "0.1.84",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.83",
+ "version": "0.1.84",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index aad2a7b47..5291367a7 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.83",
+ "version": "0.1.84",
"private": true,
"type": "module",
"scripts": {