Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
20.9 KB
Raw Blame
/**
 * Issue #654 — the LIVE wiring of the image-input guard.
 *
 * The guard itself is covered elsewhere (`core/adapters/__tests__/
 * llmPortAdapter.imageInput.test.ts`, `llm/__tests__/imageInput.test.ts`,
 * `llm/__tests__/imageInputTransport.test.ts`, `llm/__tests__/
 * claudeToolImageWire.test.ts`). This suite covers what those cannot: which
 * resolver instance actually reaches the port each production call site builds,
 * and what the resumed turn does when the capability changed while the approval
 * was pending.
 *
 * Boundary: the tool-use loop is mocked because it is the one component that
 * would otherwise execute real tools. Everything under test stays real —
 * `AgentStateMachine` → `TaskExecutor` → `LlmPortAdapter` → the provider stub.
 * Where the loop would dispatch to the model, these tests call the loop config's
 * real `reasoning` port (the production `defaultReasoningPort` the executor
 * built), which is the same object the real loop calls.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { LlmPortAdapter } from '../../core/adapters/llmPortAdapter'
import { LlmError, LlmErrorCode } from '../../core/errors'
import { executeSingleTool, runToolUseLoop } from '../../core/orchestration/toolUseLoop'
import { appendToolResults } from '../../core/orchestration/toolUseLoopMessages'
import {
  type Attachment,
  type ChatMessage,
  FinishReason,
  type ReasoningContext,
  type RespondResult,
} from '../../core/types'
import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
import type { ImageInputResolver } from '../../llm/imageInput'
import { MessageQueue } from '../../queue/messageQueue'
import type { Task, TaskError } from '../../queue/types'
import { AgentStateMachine } from '../stateMachine'

vi.mock('../../config', () => ({
  config: {
    devMode: true,
    enableApproval: true,
    enableNudge: false,
    nudgeMaxIterations: 3,
    devModelName: 'test-model',
    devModelProvider: 'openai',
    contextMaxTokens: 100000,
    tokenizerOffline: true,
    tokenizerDryrun: false,
    compactionStructuredSummary: false,
    promptCacheEnabled: false,
    nativeTool: {
      workspacePath: '/tmp',
      shellTimeout: 5000,
      httpAllowlist: [],
      envAllowlist: ['PATH'],
      memoryMaxSize: 1048576,
    },
  },
}))

vi.mock('../../core/orchestration/toolUseLoop', () => ({
  runToolUseLoop: vi.fn(),
  executeSingleTool: vi.fn(),
  validateToolLinkages: vi.fn(),
  extractInputPreview: vi.fn(() => 'ls'),
  buildOutputPreview: vi.fn((content: string) =>
    content
      ? {
          headLines: String(content).split('\n').slice(0, 3),
          tailLines: [],
          totalLines: String(content).split('\n').length,
          truncated: false,
        }
      : undefined
  ),
}))

const EVIDENCE = {
  source: 'curated' as const,
  reference: 'https://docs.z.ai/guides/vlm/glm-5.3-flash',
  checkedAt: '2026-09-16T00:00:00Z',
}

const IMAGE_B64 = 'QUJD'
const SESSION_KEY = 'user-1:rpc:wiring-session:default'

function usage() {
  return { input_tokens: 1, output_tokens: 1, total_tokens: 2 }
}

function stubProvider(type: string) {
  return {
    completeSingleTurn: vi.fn(async (_messages: ChatMessage[], _options?: unknown) => ({
      content: 'ok',
      usage: usage(),
      finish_reason: FinishReason.Stop,
    })),
    completeSingleTurnWithTools: vi.fn(
      async (_messages: ChatMessage[], _tools: unknown[], _options?: unknown) => ({
        content: 'ok',
        tool_calls: [],
        usage: usage(),
        finish_reason: FinishReason.Stop,
      })
    ),
    getProviderType: () => type,
  }
}

type StubProvider = ReturnType<typeof stubProvider>

function imageMessage(role: ChatMessage['role'] = 'user'): ChatMessage {
  return {
    role,
    content: 'look at this',
    contentParts: [
      { type: 'text', text: 'look at this' },
      { type: 'image', mimeType: 'image/png', data: IMAGE_B64 },
    ],
  }
}

function imageTask(
  channelId = 'chat-image'
): Task & { responseCallback: ReturnType<typeof vi.fn> } {
  const responseCallback = vi.fn(async () => {})
  return {
    id: `task-${channelId}-${Math.random().toString(36).slice(2, 6)}`,
    source: 'channel',
    sourceMessage: {
      sender: 'user-1',
      content: 'look at this',
      channelType: 'rpc',
      channelId,
      messageId: 'msg-1',
      timestamp: new Date().toISOString(),
      hostRef: 'test-host',
      attachments: [
        {
          id: 'att-1',
          kind: 'image',
          mimeType: 'image/png',
          encoding: 'base64',
          dataBase64: IMAGE_B64,
        },
      ],
    },
    priority: 'normal',
    status: 'pending',
    conversationHistory: [{ role: 'user', content: 'look at this', timestamp: new Date() }],
    responseCallback,
    createdAt: new Date(),
  } as unknown as Task & { responseCallback: ReturnType<typeof vi.fn> }
}

interface LoopConfigLike {
  reasoning: { respondWithTools: (context: ReasoningContext) => Promise<RespondResult> }
}

/**
 * What the real loop does with the config it is handed: it asks the reasoning
 * port for the next model turn. Calling the same object keeps the assertion on
 * the production path (reasoning port → adapter → guard → provider).
 */
function dispatchThroughReasoning(
  config: unknown,
  messages: ChatMessage[]
): Promise<RespondResult> {
  return (config as LoopConfigLike).reasoning.respondWithTools({
    messages,
    available_tools: [],
  })
}

/** The typed denial carried by a failed `RespondResult`, if any. */
function errorOf(result: RespondResult | undefined): LlmError | undefined {
  const failed = result as Extract<RespondResult, { type: 'error' }> | undefined
  return failed?.error as LlmError | undefined
}

describe('#654 image-input guard wiring', () => {
  let agent: AgentStateMachine
  let queue: MessageQueue
  let lifecycle: TaskLifecycle
  let provider: StubProvider

  beforeEach(() => {
    vi.clearAllMocks()
    vi.mocked(runToolUseLoop).mockReset()
    vi.mocked(executeSingleTool).mockReset()
    queue = new MessageQueue()
    lifecycle = new TaskLifecycle()
    queue.setLifecycle(lifecycle)
    agent = new AgentStateMachine(queue, lifecycle, { autoStart: false })
    provider = stubProvider('openai')
    agent.setLLMProvider(provider as never, 'gpt-4o')
    agent.setMcpManager({ getAllTools: () => [], callTool: vi.fn() } as never)
    agent.start()
  })

  afterEach(() => {
    vi.restoreAllMocks()
  })

  it('threads the live resolver into the port the task loop is handed', async () => {
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: { state: 'supported', evidence: EVIDENCE },
    }))
    agent.setImageInputResolver(resolver)

    let seenMessages: ChatMessage[] = []
    let seenResult: RespondResult | undefined
    vi.mocked(runToolUseLoop).mockImplementation(async (config, messages) => {
      seenMessages = messages
      seenResult = await dispatchThroughReasoning(config, messages)
      return { type: 'response', content: 'ok', usage: usage() }
    })

    const task = imageTask()
    await agent.executeTask(task)

    // The executor assembled the composer image onto the last user message…
    const last = seenMessages.at(-1)
    expect(last?.role).toBe('user')
    expect(last?.contentParts).toEqual([
      { type: 'text', text: 'look at this' },
      { type: 'image', mimeType: 'image/png', data: IMAGE_B64 },
    ])

    // …and the port behind that real reasoning call consulted the live catalog
    // for the exact pair before dispatching, instead of failing closed.
    expect(resolver).toHaveBeenCalledWith('openai', 'gpt-4o')
    expect(provider.completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
    expect(seenResult?.type).toBe('text')

    // The image survived the adapter: the provider received the content parts
    // rather than a stripped text-only message.
    const sentMessages = provider.completeSingleTurnWithTools.mock.calls[0]?.[0] as ChatMessage[]
    expect(sentMessages.at(-1)?.contentParts).toEqual([
      { type: 'text', text: 'look at this' },
      { type: 'image', mimeType: 'image/png', data: IMAGE_B64 },
    ])

    expect(task.responseCallback).toHaveBeenCalledWith(expect.objectContaining({ response: 'ok' }))
  })

  it('leaves a text-only turn untouched, even with a cold resolver', async () => {
    // A resolver that knows nothing about the pair: `unknown`, not support.
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: { state: 'unknown' },
    }))
    agent.setImageInputResolver(resolver)

    let seenResult: RespondResult | undefined
    vi.mocked(runToolUseLoop).mockImplementation(async (config, messages) => {
      seenResult = await dispatchThroughReasoning(config, messages)
      return { type: 'response', content: 'ok', usage: usage() }
    })

    const task = imageTask('chat-text')
    delete (task.sourceMessage as { attachments?: unknown }).attachments
    await agent.executeTask(task)

    expect(seenResult?.type).toBe('text')
    expect(provider.completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
    // No image in the request → the guard never consults the catalog.
    expect(resolver).not.toHaveBeenCalled()
    expect(task.responseCallback).toHaveBeenCalledWith(expect.objectContaining({ response: 'ok' }))
  })

  it('refuses an image whose evidence expired instead of dropping it into a text-only turn', async () => {
    // A validity window that has closed: `checkedAt < validUntil <= now`. The
    // adapter reads the real clock, so the window is set well in the past.
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: {
        state: 'supported',
        evidence: {
          ...EVIDENCE,
          checkedAt: '2026-01-01T00:00:00Z',
          validUntil: '2026-02-01T00:00:00Z',
        },
      },
    }))
    agent.setImageInputResolver(resolver)

    let seenResult: RespondResult | undefined
    vi.mocked(runToolUseLoop).mockImplementation(async (config, messages) => {
      seenResult = await dispatchThroughReasoning(config, messages)
      return { type: 'response', content: 'handled', usage: usage() }
    })

    await agent.executeTask(imageTask('chat-expired'))

    expect(seenResult?.type).toBe('error')
    const error = errorOf(seenResult)
    expect(error).toBeInstanceOf(LlmError)
    // Expired evidence degrades to `unknown`, never to a silent text-only send.
    expect(error?.code).toBe(LlmErrorCode.ImageInputUnknown)
    expect(error?.message).toContain('expired')
    expect(provider.completeSingleTurnWithTools).not.toHaveBeenCalled()
  })

  it('re-checks the pair when an approval resumes and never sends the image after a change', async () => {
    let state: 'supported' | 'unsupported' = 'supported'
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: { state, evidence: EVIDENCE },
    }))
    agent.setImageInputResolver(resolver)

    let resumedMessages: ChatMessage[] = []
    let resumedResult: RespondResult | undefined
    vi.mocked(runToolUseLoop)
      // First pass: suspend for approval with a snapshot that already carries
      // the image (this is what the resumed turn replays verbatim).
      .mockImplementationOnce(async () => ({
        type: 'need_approval',
        approval: {
          request_id: 'req-1',
          tool_name: 'shell_exec',
          parameters: { command: 'ls' },
          description: 'Shell command',
          tool_call_id: 'tc_1',
          context_snapshot: [imageMessage()],
        },
      }))
      // Resume: dispatch through the SAME reasoning port the executor rebuilt.
      .mockImplementationOnce(async (config, messages) => {
        resumedMessages = messages
        resumedResult = await dispatchThroughReasoning(config, messages)
        return { type: 'response', content: 'done', usage: usage() }
      })
    vi.mocked(executeSingleTool).mockResolvedValue({
      tool_call_id: 'tc_1',
      name: 'shell_exec',
      content: '<tool_output tool="shell_exec">ok</tool_output>',
      is_error: false,
    })

    const task = imageTask('chat-approval')
    await agent.executeTask(task)
    expect(agent.getState()).toBe('waiting_approval')

    // The user's model lost image support while the approval was pending.
    const resolverCallsBefore = resolver.mock.calls.length
    const providerCallsBefore = provider.completeSingleTurnWithTools.mock.calls.length
    state = 'unsupported'

    await agent.handleApproval('user-1', 'req-1', false)
    await vi.waitFor(() => expect(task.responseCallback).toHaveBeenCalled())

    // The resumed attempt re-consulted the catalog for the same pair…
    expect(resolver.mock.calls.length).toBeGreaterThan(resolverCallsBefore)
    expect(resolver).toHaveBeenCalledWith('openai', 'gpt-4o')
    // …the request really carried the image (so the refusal is about the image,
    // not about an empty request)…
    expect(resumedMessages.some(m => m.contentParts?.some(p => p.type === 'image'))).toBe(true)
    // …and the provider was never called with it.
    expect(provider.completeSingleTurnWithTools.mock.calls.length).toBe(providerCallsBefore)
    expect(resumedResult?.type).toBe('error')
    expect(errorOf(resumedResult)?.code).toBe(LlmErrorCode.ImageInputUnsupported)
    expect(task.responseCallback).toHaveBeenCalled()
  })

  it('threads the live resolver into the manual /compact port', async () => {
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: { state: 'supported', evidence: EVIDENCE },
    }))
    agent.setImageInputResolver(resolver)

    const cm = agent.getConversationManager()
    // The force-summarize tier keeps the last 5 turns, so the session needs more
    // than that before any turn is archived and the summarizer calls the port.
    const conv = await cm.getOrCreate(SESSION_KEY)
    for (let i = 0; i < 8; i++) {
      await cm.startTurn(conv, `user ${i}`, `task-${i}`)
      await cm.completeTurn(conv, `assistant ${i}`)
    }

    // Capture the adapter the compactor really uses.
    const compactPorts: LlmPortAdapter[] = []
    vi.spyOn(LlmPortAdapter.prototype, 'complete').mockImplementation(async function (
      this: LlmPortAdapter
    ) {
      if (!compactPorts.includes(this)) compactPorts.push(this)
      return {
        content: 'summary of the archived turns',
        usage: usage(),
        finish_reason: FinishReason.Stop,
      } as never
    })

    const result = await agent.compactSession({ sessionKey: SESSION_KEY })
    expect(result.kind).toBe('ok')
    expect(compactPorts).toHaveLength(1)

    // Contract note: today's compaction converts archived turns to markdown
    // text (`contextManager.ts` `formatTurnsAsMarkdown`) and sends a text-only
    // summarization request, so no archived image can reach this port. What is
    // asserted here is the WIRING: the port the compactor uses is the one
    // carrying the live resolver, and it enforces it for an image request.
    // The captured adapter is the compact port: same pair, and the injected
    // resolver is the one wired on the state machine.
    await compactPorts[0].completeWithTools({ messages: [imageMessage()], tools: [] })
    expect(resolver).toHaveBeenCalledWith('openai', 'gpt-4o')
    expect(provider.completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
  })

  it('keeps an unwired compact port failing closed on images but not on text', async () => {
    // No `setImageInputResolver` call at all: a cold host.
    const cm = agent.getConversationManager()
    const conv = await cm.getOrCreate(SESSION_KEY)
    for (let i = 0; i < 8; i++) {
      await cm.startTurn(conv, `user ${i}`, `task-${i}`)
      await cm.completeTurn(conv, `assistant ${i}`)
    }

    const compactPorts: LlmPortAdapter[] = []
    vi.spyOn(LlmPortAdapter.prototype, 'complete').mockImplementation(async function (
      this: LlmPortAdapter
    ) {
      if (!compactPorts.includes(this)) compactPorts.push(this)
      return {
        content: 'summary of the archived turns',
        usage: usage(),
        finish_reason: FinishReason.Stop,
      } as never
    })

    // Compaction itself is text-only, so a cold catalog must not break it.
    const result = await agent.compactSession({ sessionKey: SESSION_KEY })
    expect(result.kind).toBe('ok')
    expect(compactPorts).toHaveLength(1)

    await expect(
      compactPorts[0].completeWithTools({ messages: [imageMessage()], tools: [] })
    ).rejects.toMatchObject({
      code: LlmErrorCode.ImageInputUnknown,
      retryable: false,
    })
    expect(provider.completeSingleTurnWithTools).not.toHaveBeenCalled()
  })

  it('a screenshot tool does not terminate a text-only task on an unverified model', async () => {
    // A cold catalog: the pair resolves to `unknown`, which refuses a USER
    // image. The task below never carries one — the image comes back from the
    // agent's own tool, and that must not end the task.
    const resolver = vi.fn<ImageInputResolver>(() => ({
      capability: { state: 'unknown' },
    }))
    agent.setImageInputResolver(resolver)

    const collectedAttachments: Attachment[] = []
    let seenResult: RespondResult | undefined
    vi.mocked(runToolUseLoop).mockImplementation(async (config, messages) => {
      // The real message builder, so the `imageOrigin` flag under test is the
      // one production writes rather than one this test invents.
      appendToolResults(
        messages,
        [
          {
            tool_call_id: 'tc_1',
            name: 'browser__screenshot',
            content: 'captured the page',
            is_error: false,
            attachments: [
              {
                id: 'shot-1',
                kind: 'image',
                mimeType: 'image/png',
                encoding: 'base64',
                dataBase64: IMAGE_B64,
                sourceTool: 'browser__screenshot',
              } as unknown as Attachment,
            ],
          },
        ],
        collectedAttachments
      )
      seenResult = await dispatchThroughReasoning(config, messages)
      return { type: 'response', content: 'ok', usage: usage(), attachments: collectedAttachments }
    })

    const task = imageTask('chat-tool-shot')
    delete (task.sourceMessage as { attachments?: unknown }).attachments
    await agent.executeTask(task)

    // Witness: the turn reached the model and came back as text, not as a
    // typed image refusal.
    expect(seenResult?.type).toBe('text')
    expect(errorOf(seenResult)).toBeUndefined()
    expect(provider.completeSingleTurnWithTools).toHaveBeenCalledTimes(1)
    expect(resolver).toHaveBeenCalledWith('openai', 'gpt-4o')

    // The screenshot was withheld from the wire and replaced by the notice.
    const sent = provider.completeSingleTurnWithTools.mock.calls[0]?.[0] as ChatMessage[]
    const screenshotMessage = sent.at(-1)
    expect(screenshotMessage?.contentParts).toBeUndefined()
    expect(screenshotMessage?.content).toContain('were not forwarded')

    // …and it still reaches the user as an attachment on the reply: withholding
    // is about what the MODEL can read, not about discarding the bytes.
    expect(collectedAttachments).toHaveLength(1)
    expect(task.responseCallback).toHaveBeenCalledWith(
      expect.objectContaining({
        response: 'ok',
        attachments: [expect.objectContaining({ id: 'shot-1', kind: 'image' })],
      })
    )
  })
})

/**
 * The decision must also surface as a structured task error when it reaches the
 * executor's failure path, so the channel reader sees a code it can act on.
 */
describe('#654 image-input denial is a structured task error', () => {
  let agent: AgentStateMachine
  let provider: StubProvider

  beforeEach(() => {
    vi.clearAllMocks()
    vi.mocked(runToolUseLoop).mockReset()
    vi.mocked(executeSingleTool).mockReset()
    const queue = new MessageQueue()
    const lifecycle = new TaskLifecycle()
    queue.setLifecycle(lifecycle)
    agent = new AgentStateMachine(queue, lifecycle, { autoStart: false })
    provider = stubProvider('openai')
    agent.setLLMProvider(provider as never, 'gpt-4o')
    agent.setMcpManager({ getAllTools: () => [], callTool: vi.fn() } as never)
    agent.start()
  })

  afterEach(() => {
    vi.restoreAllMocks()
  })

  it('delivers LLM_IMAGE_INPUT_UNSUPPORTED to the response callback', async () => {
    agent.setImageInputResolver(() => ({
      capability: { state: 'unsupported', evidence: EVIDENCE },
    }))

    // The real loop turns a reasoning error into the loop's error result.
    vi.mocked(runToolUseLoop).mockImplementation(async config => {
      const result = await dispatchThroughReasoning(config, [imageMessage()])
      if (result.type === 'error') throw result.error
      return { type: 'response', content: 'unexpected', usage: usage() }
    })

    const task = imageTask('chat-task-error')
    await agent.executeTask(task)

    expect(task.responseCallback).toHaveBeenCalledTimes(1)
    const delivered = task.responseCallback.mock.calls[0][0] as { error?: TaskError }
    expect(delivered.error?.code).toBe(LlmErrorCode.ImageInputUnsupported)
    expect(delivered.error?.retryable).toBe(false)
    expect(delivered.error?.provider).toBe('openai')
    expect(provider.completeSingleTurnWithTools).not.toHaveBeenCalled()
  })
})