Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
13.9 KB
Raw Blame
/**
 * Issue #654 — cross-suite guard for the transport matrix.
 *
 * Uses the REAL provider implementations with their SDK clients intercepted, so
 * every cell the guard labels image-capable is proven by the serializer that
 * actually runs. A cell that says "supported" but drops the part on the wire, or
 * an "unsupported" cell that silently reaches the SDK, fails here.
 */
import { describe, expect, it, vi } from 'vitest'
import { LlmPortAdapter } from '../../core/adapters/llmPortAdapter'
import { LlmError, LlmErrorCode } from '../../core/errors'
import type { SystemPromptParts } from '../../core/reasoning/systemPrompt'
import { type ChatMessage, FinishReason } from '../../core/types'
import { ClaudeProvider } from '../claude'
import { type BedrockConverseClient, BedrockConverseDriver } from '../drivers/bedrockConverse'
import { type GeminiGenerateClient, GoogleGenerativeDriver } from '../drivers/googleGenerative'
import type { ImageInputResolver, ImageTransportOperation } from '../imageInput'
import { OpenAIProvider } from '../openai'
import { OpenAICompatibleProvider } from '../openaiCompatible'
import type { ClassifiedError, SingleTurnProvider } from '../types'

const EVIDENCE = {
  source: 'curated' as const,
  reference: 'https://docs.z.ai/guides/vlm/glm-5.3-flash',
  checkedAt: '2026-09-16T00:00:00Z',
}

const allowResolver: ImageInputResolver = () => ({
  capability: { state: 'supported', evidence: EVIDENCE },
})

function parts(): SystemPromptParts {
  return { stable: 'stable', context: 'context', stableHash: 'h1', contextHash: 'h2' }
}

function imageMessage(): ChatMessage {
  return {
    role: 'user',
    content: 'what does this picture show?',
    contentParts: [
      { type: 'text', text: 'what does this picture show?' },
      { type: 'image', mimeType: 'image/png', data: 'QUJD' },
    ],
  }
}

const OPENAI_OK = {
  choices: [{ message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
  usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
}
const CLAUDE_OK = {
  content: [{ type: 'text', text: 'ok' }],
  stop_reason: 'end_turn',
  usage: { input_tokens: 1, output_tokens: 1 },
}
const GEMINI_OK = {
  candidates: [{ content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' }],
  usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1, totalTokenCount: 2 },
}
const BEDROCK_OK = {
  output: { message: { content: [{ text: 'ok' }] } },
  stopReason: 'end_turn',
  usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
}

type Family = 'openai-compatible' | 'claude' | 'vertex' | 'bedrock' | 'codex'

interface Harness {
  provider: SingleTurnProvider
  /** Number of SDK calls that left the process boundary. */
  sdkCalls: () => number
  /** The last wire payload handed to the SDK, or undefined. */
  wire: () => unknown
}

function buildHarness(family: Family): Harness {
  switch (family) {
    case 'openai-compatible': {
      const create = vi.fn(async (_input: unknown) => OPENAI_OK)
      const provider = new OpenAIProvider({ chat: { completions: { create } } } as never, 'gpt-4o')
      return {
        provider,
        sdkCalls: () => create.mock.calls.length,
        wire: () => create.mock.calls[0]?.[0],
      }
    }
    case 'claude': {
      const create = vi.fn(async (_input: unknown) => CLAUDE_OK)
      const provider = new ClaudeProvider({ messages: { create } } as never, 'claude-sonnet-4-6')
      return {
        provider,
        sdkCalls: () => create.mock.calls.length,
        wire: () => create.mock.calls[0]?.[0],
      }
    }
    case 'vertex': {
      const generateContent = vi.fn(async (_input: unknown) => GEMINI_OK)
      const client = { generateContent } as unknown as GeminiGenerateClient
      const provider = new GoogleGenerativeDriver(client, 'gemini-2.5-pro')
      return {
        provider,
        sdkCalls: () => generateContent.mock.calls.length,
        wire: () => generateContent.mock.calls[0]?.[0],
      }
    }
    case 'bedrock': {
      const converse = vi.fn(async (_input: unknown) => BEDROCK_OK)
      const client = { converse } as unknown as BedrockConverseClient
      const provider = new BedrockConverseDriver(client, 'anthropic.claude-sonnet-4-6-v1:0')
      return {
        provider,
        sdkCalls: () => converse.mock.calls.length,
        wire: () => converse.mock.calls[0]?.[0],
      }
    }
    case 'codex': {
      const textOk = {
        content: 'ok',
        usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
        finish_reason: FinishReason.Stop,
      }
      let lastMessages: ChatMessage[] | undefined
      const withTools = vi.fn(async (messages: ChatMessage[]) => {
        lastMessages = messages
        return { ...textOk, tool_calls: [] }
      })
      const single = vi.fn(async (messages: ChatMessage[]) => {
        lastMessages = messages
        return textOk
      })
      const provider = {
        completeSingleTurn: single,
        completeSingleTurnWithTools: withTools,
        getProviderType: () => 'codex-subscription' as const,
        classifyError: (err: unknown): ClassifiedError => ({
          code: LlmErrorCode.ApiCallFailed,
          retryable: true,
          message: (err as Error).message,
        }),
      } as unknown as SingleTurnProvider
      return {
        provider,
        sdkCalls: () => single.mock.calls.length + withTools.mock.calls.length,
        wire: () => lastMessages,
      }
    }
  }
}

/** Structural image-presence check per wire family. */
function wireHasImage(family: Family, wire: unknown): boolean {
  const payload = wire as {
    messages?: Array<{ content?: unknown }>
    contents?: Array<{ parts?: Array<{ inlineData?: unknown }> }>
  }
  switch (family) {
    case 'openai-compatible':
      return Boolean(
        payload.messages?.some(
          m =>
            Array.isArray(m.content) &&
            (m.content as Array<{ type?: string }>).some(p => p.type === 'image_url')
        )
      )
    case 'claude':
      return Boolean(
        payload.messages?.some(
          m =>
            Array.isArray(m.content) &&
            (m.content as Array<{ type?: string }>).some(p => p.type === 'image')
        )
      )
    case 'vertex':
      return Boolean(payload.contents?.some(c => c.parts?.some(p => p.inlineData !== undefined)))
    case 'bedrock':
      return Boolean(
        payload.messages?.some(
          m =>
            Array.isArray(m.content) &&
            (m.content as Array<Record<string, unknown>>).some(b => 'image' in b)
        )
      )
    case 'codex':
      return Boolean(
        Array.isArray(wire) &&
        wire.some((message: ChatMessage) =>
          message.contentParts?.some(part => part.type === 'image')
        )
      )
  }
}

/** Representative REGISTERED provider id for each family (never a family name). */
function providerIdFor(family: Family): string {
  switch (family) {
    case 'openai-compatible':
      return 'openai'
    case 'claude':
      return 'claude'
    case 'vertex':
      return 'vertex'
    case 'bedrock':
      return 'bedrock'
    case 'codex':
      return 'codex-subscription'
  }
}

function modelFor(family: Family): string {
  switch (family) {
    case 'openai-compatible':
      return 'gpt-4o'
    case 'claude':
      return 'claude-sonnet-4-6'
    case 'vertex':
      return 'gemini-2.5-pro'
    case 'bedrock':
      return 'anthropic.claude-sonnet-4-6-v1:0'
    case 'codex':
      return 'gpt-5.3-codex'
  }
}

function adapterFor(harness: Harness, family: Family): LlmPortAdapter {
  return new LlmPortAdapter(
    harness.provider,
    modelFor(family),
    providerIdFor(family),
    undefined,
    undefined,
    undefined,
    undefined,
    undefined,
    allowResolver
  )
}

/** Dispatch the operation the way the real call sites do. */
async function dispatch(
  adapter: LlmPortAdapter,
  operation: ImageTransportOperation,
  messages: ChatMessage[]
): Promise<void> {
  switch (operation) {
    case 'complete':
      await adapter.complete({ messages })
      return
    case 'completeWithTools':
      await adapter.completeWithTools({ messages, tools: [] })
      return
    case 'completeAndCache':
      await adapter.complete({ messages, systemPromptParts: parts() })
      return
    case 'completeWithToolsAndCache':
      await adapter.completeWithTools({ messages, tools: [], systemPromptParts: parts() })
  }
}

/** Cells each family can really reach through the adapter. */
const REACHABLE: Record<Family, ImageTransportOperation[]> = {
  'openai-compatible': ['complete', 'completeWithTools'],
  claude: ['complete', 'completeWithTools', 'completeAndCache', 'completeWithToolsAndCache'],
  vertex: ['complete', 'completeWithTools'],
  bedrock: ['complete', 'completeWithTools'],
  codex: ['complete', 'completeWithTools'],
}

/** Expected outcome, restated from plan §4.1 (not imported from the module). */
function expectedSupported(family: Family, operation: ImageTransportOperation): boolean {
  if (family === 'codex') return operation === 'complete' || operation === 'completeWithTools'
  if (operation === 'complete') return family === 'vertex' || family === 'bedrock'
  if (operation === 'completeWithTools') return true
  return family === 'claude'
}

const FAMILIES: Family[] = ['openai-compatible', 'claude', 'vertex', 'bedrock', 'codex']

describe('#654 transport cross-suite (real serializers)', () => {
  for (const family of FAMILIES) {
    for (const operation of REACHABLE[family]) {
      const supported = expectedSupported(family, operation)
      it(`${family}/${operation} → ${supported ? 'keeps the image on the wire' : 'never reaches the SDK'}`, async () => {
        const harness = buildHarness(family)
        const adapter = adapterFor(harness, family)

        if (supported) {
          await dispatch(adapter, operation, [imageMessage()])
          expect(harness.sdkCalls()).toBe(1)
          expect(wireHasImage(family, harness.wire())).toBe(true)
        } else {
          await expect(dispatch(adapter, operation, [imageMessage()])).rejects.toBeInstanceOf(
            LlmError
          )
          expect(harness.sdkCalls()).toBe(0)
        }
      })
    }
  }

  it('text-only traffic is dispatched unchanged on every family', async () => {
    for (const family of FAMILIES) {
      const harness = buildHarness(family)
      const adapter = adapterFor(harness, family)
      await adapter.completeWithTools({ messages: [{ role: 'user', content: 'hi' }], tools: [] })
      expect(harness.sdkCalls()).toBe(1)
    }
  })

  it('counterfactual: the tool-less path really drops the part (so the cell is not a guess)', async () => {
    // OpenAI: completeSingleTurn rebuilds role/content and loses contentParts.
    const openaiHarness = buildHarness('openai-compatible')
    await openaiHarness.provider.completeSingleTurn([imageMessage()])
    expect(openaiHarness.sdkCalls()).toBe(1)
    expect(wireHasImage('openai-compatible', openaiHarness.wire())).toBe(false)

    // Claude: the tool-less variant does the same.
    const claudeHarness = buildHarness('claude')
    await claudeHarness.provider.completeSingleTurn([imageMessage()])
    expect(claudeHarness.sdkCalls()).toBe(1)
    expect(wireHasImage('claude', claudeHarness.wire())).toBe(false)
  })

  it('counterfactual: the tool-bearing paths really preserve the part', async () => {
    for (const family of ['openai-compatible', 'claude', 'vertex', 'bedrock'] as Family[]) {
      const harness = buildHarness(family)
      await harness.provider.completeSingleTurnWithTools([imageMessage()], [])
      expect(harness.sdkCalls()).toBe(1)
      expect(wireHasImage(family, harness.wire())).toBe(true)
    }
  })

  it('names the pair that would have been called in the refusal', async () => {
    const harness = buildHarness('openai-compatible')
    const adapter = adapterFor(harness, 'openai-compatible')
    await expect(adapter.complete({ messages: [imageMessage()] })).rejects.toMatchObject({
      code: LlmErrorCode.ImageInputUnsupported,
      retryable: false,
      provider: 'openai',
    })
  })

  it('does not leak image bytes or base64 into the refusal message', async () => {
    const harness = buildHarness('openai-compatible')
    const adapter = new LlmPortAdapter(
      harness.provider,
      'glm-5.3',
      'zai',
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      () => ({
        capability: { state: 'unsupported', evidence: EVIDENCE },
      })
    )
    let error: LlmError | undefined
    try {
      await adapter.completeWithTools({ messages: [imageMessage()], tools: [] })
    } catch (err) {
      error = err as LlmError
    }
    expect(error).toBeInstanceOf(LlmError)
    expect(error!.message).not.toContain('QUJD')
    expect(error!.message).not.toContain('base64')
  })

  it('the data-driven compatible arm shares the proven serializer (real subclass, stub transport)', async () => {
    // `OpenAICompatibleProvider` builds its own SDK client, so the client field is
    // swapped for a stub AFTER construction: the class, its provider id and every
    // serializer it runs are the production ones.
    const provider = new OpenAICompatibleProvider(
      { id: 'zai', baseURL: 'https://api.z.ai/api/coding/paas/v4', defaultModel: 'glm-5.3-flash' },
      'fake-key',
      'glm-5.3-flash'
    )
    const create = vi.fn(async (_input: unknown) => OPENAI_OK)
    ;(provider as unknown as { client: unknown }).client = {
      chat: { completions: { create } },
    }
    const adapter = new LlmPortAdapter(
      provider,
      'glm-5.3-flash',
      'zai',
      undefined,
      undefined,
      undefined,
      undefined,
      undefined,
      allowResolver
    )

    await adapter.completeWithTools({ messages: [imageMessage()], tools: [] })
    expect(create).toHaveBeenCalledTimes(1)
    expect(wireHasImage('openai-compatible', create.mock.calls[0]?.[0])).toBe(true)

    // The tool-less path is still refused, so Flash cannot be attached for a
    // `complete` call by accident.
    await expect(adapter.complete({ messages: [imageMessage()] })).rejects.toBeInstanceOf(LlmError)
  })
})