Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
22.1 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest'
import { LlmErrorCode } from '../../core/errors'
import { FinishReason, type ToolCompletionResponse } from '../../core/types'
import type { SingleTurnProvider } from '../../llm'
import type { McpClientConnection, McpClientFactory } from '../stepRouter'
import { WorkflowService } from '../workflowService'

// ─── Mock Factories ─────────────────────────────────────────────────────

function mockLlmProvider(
  responses: Array<{
    content: string | null
    tool_calls?: Array<{ name: string; arguments: Record<string, unknown> }> | null
    finish_reason?: FinishReason | string
  }>
): SingleTurnProvider {
  let callIndex = 0
  return {
    completeSingleTurn: vi.fn().mockImplementation(() => {
      const r = responses[callIndex++] ?? { content: 'done' }
      return Promise.resolve({
        content: r.content ?? '',
        usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
        finish_reason: r.finish_reason ?? 'stop',
      })
    }),
    completeSingleTurnWithTools: vi.fn().mockImplementation(() => {
      const r = responses[callIndex++] ?? { content: 'done', tool_calls: null }
      return Promise.resolve({
        content: r.content,
        tool_calls: r.tool_calls ?? null,
        usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
        finish_reason: r.finish_reason ?? (r.tool_calls ? 'tool_use' : 'stop'),
      })
    }),
    getProviderType: () => 'openai',
    classifyError: vi.fn(() => ({
      code: LlmErrorCode.ApiCallFailed,
      retryable: true,
      message: 'mock',
    })),
  }
}

afterEach(() => {
  delete process.env.MCP_HOST_WORKFLOW_MAX_OUTPUT_TOKENS
})

function mockLlmFactory(provider: SingleTurnProvider) {
  return () => provider
}

function mockMcpClient(
  tools: Array<{ name: string }> = [],
  callResults: Record<string, unknown> = {}
): McpClientConnection {
  return {
    connect: vi.fn().mockResolvedValue(undefined),
    listTools: vi.fn().mockResolvedValue(tools.map(t => ({ name: t.name, description: '' }))),
    callTool: vi
      .fn()
      .mockImplementation((name: string) =>
        Promise.resolve({ content: callResults[name] ?? `result-${name}`, isError: false })
      ),
    disconnect: vi.fn().mockResolvedValue(undefined),
  }
}

function mockMcpFactory(clients: Map<string, McpClientConnection>): McpClientFactory {
  return server => clients.get(server.name) ?? mockMcpClient()
}

function createConfiguredService(
  llmProvider: SingleTurnProvider,
  mcpClients?: Map<string, McpClientConnection>
) {
  const svc = new WorkflowService('test-recipe', {
    llmFactory: mockLlmFactory(llmProvider),
    mcpClientFactory: mcpClients ? mockMcpFactory(mcpClients) : () => mockMcpClient(),
  })
  svc.configure({ provider: 'openai', model: 'gpt-4', apiKey: 'sk-123' })
  return svc
}

describe('POST /execute — happy path', () => {
  it('connects to declared MCP servers', async () => {
    const client = mockMcpClient([{ name: 'tool1' }])
    const llm = mockLlmProvider([{ content: 'done', tool_calls: null }])
    const svc = createConfiguredService(llm, new Map([['srv', client]]))

    await svc.executeStep({
      stepId: 's1',
      instruction: 'do stuff',
      mcpServers: [{ name: 'srv', url: 'http://srv' }],
    })
    expect(client.connect).toHaveBeenCalled()
  })

  it('calls LLM with filtered tool list', async () => {
    const llm = mockLlmProvider([{ content: 'ok', tool_calls: null }])
    const client = mockMcpClient([{ name: 'read' }, { name: 'write' }])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    await svc.executeStep({
      stepId: 's1',
      instruction: 'query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
    })

    expect(llm.completeSingleTurnWithTools).toHaveBeenCalled()
  })

  it('passes caller abort signal into the LLM provider', async () => {
    let providerSignal: AbortSignal | undefined
    let markProviderCalled!: () => void
    const providerCalled = new Promise<void>(resolve => {
      markProviderCalled = resolve
    })
    const completeSingleTurnWithTools = vi.fn(
      (
        _messages: Parameters<SingleTurnProvider['completeSingleTurnWithTools']>[0],
        _tools: Parameters<SingleTurnProvider['completeSingleTurnWithTools']>[1],
        options?: Parameters<SingleTurnProvider['completeSingleTurnWithTools']>[2]
      ): Promise<ToolCompletionResponse> =>
        new Promise<ToolCompletionResponse>((_resolve, reject) => {
          providerSignal = options?.signal
          markProviderCalled()
          if (options?.signal?.aborted) {
            reject(new Error(String(options.signal.reason ?? 'aborted')))
            return
          }
          options?.signal?.addEventListener(
            'abort',
            () => reject(new Error(String(options.signal?.reason ?? 'aborted'))),
            { once: true }
          )
        })
    )
    const llm: SingleTurnProvider = {
      completeSingleTurn: vi.fn(),
      completeSingleTurnWithTools,
      getProviderType: () => 'openai',
      classifyError: vi.fn(() => ({
        code: LlmErrorCode.ApiCallFailed,
        retryable: true,
        message: 'mock',
      })),
    }
    const client = mockMcpClient([{ name: 'read' }])
    const svc = createConfiguredService(llm, new Map([['db', client]]))
    const controller = new AbortController()

    const pending = svc.executeStep(
      {
        stepId: 's1',
        instruction: 'query',
        mcpServers: [{ name: 'db', url: 'http://db' }],
      },
      undefined,
      { signal: controller.signal }
    )
    await providerCalled
    controller.abort('client-disconnected')
    const result = await pending

    expect(result.status).toBe('failed')
    expect(result.error).toBe('client-disconnected')
    expect(providerSignal).toBeInstanceOf(AbortSignal)
  })

  it('dispatches tool calls returned by LLM', async () => {
    const client = mockMcpClient([{ name: 'read' }])
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'db__read', arguments: { q: 'SELECT 1' } }] },
      { content: 'Query result: 1', tool_calls: null },
    ])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
    })

    expect(client.callTool).toHaveBeenCalledWith(
      'read',
      { q: 'SELECT 1' },
      expect.objectContaining({
        signal: expect.any(AbortSignal),
        timeoutMs: expect.any(Number),
      })
    )
    const callOptions = (client.callTool as ReturnType<typeof vi.fn>).mock.calls[0]?.[2]
    expect(callOptions?.signal.aborted).toBe(false)
    expect(callOptions?.timeoutMs).toBeGreaterThan(0)
    expect(result.status).toBe('completed')
  })

  it('continues LLM loop until no tool calls in response', async () => {
    const client = mockMcpClient([{ name: 't1' }])
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'srv__t1', arguments: {} }] },
      { content: null, tool_calls: [{ name: 'srv__t1', arguments: {} }] },
      { content: 'final answer', tool_calls: null },
    ])
    const svc = createConfiguredService(llm, new Map([['srv', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'loop test',
      mcpServers: [{ name: 'srv', url: 'http://srv' }],
    })

    expect(result.output).toBe('final answer')
    expect(result.toolsCalled).toHaveLength(2)
  })

  it('returns StepExecutionResult with output, toolsCalled, tokensUsed', async () => {
    const llm = mockLlmProvider([{ content: 'done', tool_calls: null }])
    const svc = createConfiguredService(llm)

    const result = await svc.executeStep({ stepId: 's1', instruction: 'test' })
    expect(result.stepId).toBe('s1')
    expect(result.status).toBe('completed')
    expect(result.output).toBe('done')
    expect(result.tokensUsed).toBeDefined()
  })

  it('includes completedAt in ISO 8601 format', async () => {
    const llm = mockLlmProvider([{ content: 'done' }])
    const svc = createConfiguredService(llm)

    const result = await svc.executeStep({ stepId: 's1', instruction: 'test' })
    expect(result.completedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/)
  })

  it('passes MCP_HOST_WORKFLOW_MAX_OUTPUT_TOKENS to text-only provider calls when configured', async () => {
    process.env.MCP_HOST_WORKFLOW_MAX_OUTPUT_TOKENS = '2048'
    const llm = mockLlmProvider([{ content: 'done', tool_calls: null }])
    const svc = createConfiguredService(llm)

    await svc.executeStep({ stepId: 's1', instruction: 'test' })

    expect(llm.completeSingleTurn).toHaveBeenCalledWith(
      expect.any(Array),
      expect.objectContaining({ max_tokens: 2048 })
    )
  })

  it('passes MCP_HOST_WORKFLOW_MAX_OUTPUT_TOKENS to tool-capable provider calls when configured', async () => {
    process.env.MCP_HOST_WORKFLOW_MAX_OUTPUT_TOKENS = '1024'
    const llm = mockLlmProvider([{ content: 'ok', tool_calls: null }])
    const svc = createConfiguredService(llm)

    await svc.executeStep({
      stepId: 's1',
      instruction: 'generate report',
      allowedTools: { include: ['clerum__generate_pdf'] },
    })

    expect(llm.completeSingleTurnWithTools).toHaveBeenCalledWith(
      expect.any(Array),
      expect.any(Array),
      expect.objectContaining({ max_tokens: 1024 })
    )
  })

  it('retries once when provider stops by length and completes with continuation output', async () => {
    const llm = mockLlmProvider([
      { content: 'part one', finish_reason: FinishReason.Length },
      { content: 'part two', finish_reason: FinishReason.Stop },
    ])
    const svc = createConfiguredService(llm)

    const result = await svc.executeStep({ stepId: 's1', instruction: 'write report' })

    expect(result.status).toBe('completed')
    expect(result.output).toBe('part one\npart two')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(2)
  })

  it('fails explicitly when provider remains length-limited without an artifact', async () => {
    const llm = mockLlmProvider([
      { content: 'part one', finish_reason: FinishReason.Length },
      { content: 'part two', finish_reason: FinishReason.Length },
    ])
    const svc = createConfiguredService(llm)

    const result = await svc.executeStep({ stepId: 's1', instruction: 'write report' })

    expect(result.status).toBe('failed')
    expect(result.error).toContain('provider_output_length_exceeded')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(2)
  })

  it('keeps artifact source of truth when provider remains length-limited after creating an artifact', async () => {
    const artifact = {
      success: true,
      artifact: {
        name: 'report.pdf',
        format: 'pdf',
        path: '/output/report.pdf',
        sizeBytes: 12,
        createdAt: '2026-05-13T00:00:00.000Z',
      },
    }
    const client = mockMcpClient([{ name: 'make_report' }], { make_report: artifact })
    const llm = mockLlmProvider([
      {
        content: null,
        tool_calls: [
          {
            name: 'srv__make_report',
            arguments: { filename: 'report.pdf' },
          },
        ],
      },
      { content: 'partial summary', finish_reason: FinishReason.Length },
      { content: 'still partial', finish_reason: FinishReason.Length },
    ])
    const svc = createConfiguredService(llm, new Map([['srv', client]]))

    const result = await svc.executeStep({
      stepId: 'generate-report',
      instruction: 'generate a report',
      mcpServers: [{ name: 'srv', url: 'http://srv' }],
    })

    expect(result.status).toBe('completed')
    expect(result.output).toContain('Use the run artifacts for the complete output')
    expect(result.output).toContain('report.pdf')
    expect(result.toolsCalled).toHaveLength(1)
  })
})

describe('POST /execute — tool scoping', () => {
  it('uses text-only LLM calls for steps without MCP servers or explicit allowedTools', async () => {
    const llm = mockLlmProvider([{ content: 'analysis complete', tool_calls: null }])
    const svc = createConfiguredService(llm)

    await svc.executeStep({
      stepId: 'analysis',
      instruction: 'summarize the prior output',
    })

    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
    expect(llm.completeSingleTurnWithTools).not.toHaveBeenCalled()
  })

  it('presents internal tools only when explicitly allowed by the step', async () => {
    const llm = mockLlmProvider([{ content: 'ok', tool_calls: null }])
    const svc = createConfiguredService(llm)

    await svc.executeStep({
      stepId: 'generate-report',
      instruction: 'generate a report',
      allowedTools: { include: ['clerum__generate_pdf'] },
    })

    const callArgs = (llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mock.calls[0]
    const tools = callArgs[1]
    expect(tools).toHaveLength(1)
    expect(tools[0].name).toBe('clerum__generate_pdf')
  })

  it('presents only allowedTools.include tools to LLM', async () => {
    const client = mockMcpClient([{ name: 'read' }, { name: 'write' }, { name: 'delete' }])
    const llm = mockLlmProvider([{ content: 'ok', tool_calls: null }])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    await svc.executeStep({
      stepId: 's1',
      instruction: 'safe query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
    })

    const callArgs = (llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mock.calls[0]
    const tools = callArgs[1]
    expect(tools).toHaveLength(1)
    expect(tools[0].name).toBe('db__read')
  })

  it('forwards explicit toolChoice to the tool-calling provider call', async () => {
    const client = mockMcpClient([{ name: 'read' }])
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'db__read', arguments: { id: 'one' } }] },
      { content: 'ok', tool_calls: null },
    ])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    await svc.executeStep({
      stepId: 's1',
      instruction: 'safe query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
      toolChoice: 'required',
      maxIterations: 2,
    })

    const callArgs = (llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mock.calls[0]
    expect(callArgs[2]).toEqual(expect.objectContaining({ tool_choice: 'required' }))
  })

  it('fails explicit toolChoice=required when the provider never emits a structured tool call', async () => {
    const client = mockMcpClient([{ name: 'read' }])
    const llm = mockLlmProvider([
      { content: 'plain text is not enough', tool_calls: null },
      { content: 'still no tool call', tool_calls: null },
    ])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'safe query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
      toolChoice: 'required',
      maxIterations: 2,
    })

    expect(result.status).toBe('failed')
    expect(result.error).toContain('required-tool-call-not-made')
    expect(client.callTool).not.toHaveBeenCalled()
    expect(llm.completeSingleTurnWithTools).toHaveBeenCalledTimes(2)
    const calls = (llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mock.calls
    expect(calls[0][1]).toHaveLength(1)
    expect(calls[1][1]).toHaveLength(1)
    expect(calls[1][2]).toEqual(expect.objectContaining({ tool_choice: 'required' }))
  })

  it('fails explicit toolChoice=required when the required tool returns an error result', async () => {
    const client = mockMcpClient([{ name: 'read' }], {
      read: { success: false, error: 'broker denied the workflow trigger' },
    })
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'db__read', arguments: { id: 'one' } }] },
      { content: 'done anyway', tool_calls: null },
    ])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'safe query',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
      toolChoice: 'required',
      maxIterations: 2,
    })

    expect(result.status).toBe('failed')
    expect(result.error).toContain('required-tool-call-failed')
    expect(result.error).toContain('broker denied the workflow trigger')
    expect(client.callTool).toHaveBeenCalledWith(
      'read',
      { id: 'one' },
      expect.objectContaining({
        signal: expect.any(Object),
        timeoutMs: expect.any(Number),
      })
    )
  })

  it('does not present disallowed tools even when server exposes them', async () => {
    const client = mockMcpClient([{ name: 'read' }, { name: 'delete' }])
    const llm = mockLlmProvider([{ content: 'ok', tool_calls: null }])
    const svc = createConfiguredService(llm, new Map([['db', client]]))

    await svc.executeStep({
      stepId: 's1',
      instruction: 'safe',
      mcpServers: [{ name: 'db', url: 'http://db' }],
      allowedTools: { include: ['db__read'] },
    })

    const tools = (llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mock.calls[0][1]
    expect(tools.some((t: { name: string }) => t.name === 'db__delete')).toBe(false)
  })
})

describe('POST /execute — connection failure', () => {
  it('returns failed with error when MCP connect fails', async () => {
    const client = mockMcpClient()
    ;(client.connect as ReturnType<typeof vi.fn>).mockRejectedValue(new Error('refused'))
    const llm = mockLlmProvider([])
    const svc = createConfiguredService(llm, new Map([['bad', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'test',
      mcpServers: [{ name: 'bad', url: 'http://bad' }],
    })
    expect(result.status).toBe('failed')
  })

  it('calls disconnect in finally block on connection failure', async () => {
    // disconnect is called inside the StepMcpRouter — tested in stepRouter.test.ts
    // Here we verify the service doesn't throw uncaught
    const client = mockMcpClient()
    ;(client.connect as ReturnType<typeof vi.fn>).mockRejectedValue(new Error('refused'))
    const llm = mockLlmProvider([])
    const svc = createConfiguredService(llm, new Map([['bad', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'test',
      mcpServers: [{ name: 'bad', url: 'http://bad' }],
    })
    expect(result.status).toBe('failed')
  })
})

describe('POST /execute — SOUL lifecycle', () => {
  it('uses per-step SOUL when soulOverrideActive is true', async () => {
    const llm = mockLlmProvider([{ content: 'ok' }])
    const svc = new WorkflowService('test', {
      llmFactory: mockLlmFactory(llm),
      mcpClientFactory: () => mockMcpClient(),
      globalSoulContent: 'Global SOUL',
    })
    svc.configure({ provider: 'openai', model: 'gpt-4', apiKey: 'k', soulContent: 'Step SOUL' })

    expect(svc.getActiveSoulContent()).toBe('Step SOUL')
  })

  it('reverts to global SOUL after step completes successfully', async () => {
    const llm = mockLlmProvider([{ content: 'ok' }])
    const svc = new WorkflowService('test', {
      llmFactory: mockLlmFactory(llm),
      mcpClientFactory: () => mockMcpClient(),
      globalSoulContent: 'Global SOUL',
    })
    svc.configure({ provider: 'openai', model: 'gpt-4', apiKey: 'k', soulContent: 'Step SOUL' })

    await svc.executeStep({ stepId: 's1', instruction: 'test' })

    expect(svc.isSoulOverrideActive()).toBe(false)
    expect(svc.getActiveSoulContent()).toBe('Global SOUL')
  })

  it('reverts to global SOUL after step fails', async () => {
    const client = mockMcpClient()
    ;(client.connect as ReturnType<typeof vi.fn>).mockRejectedValue(new Error('fail'))
    const llm = mockLlmProvider([])
    const svc = new WorkflowService('test', {
      llmFactory: mockLlmFactory(llm),
      mcpClientFactory: () => client,
      globalSoulContent: 'Global',
    })
    svc.configure({ provider: 'openai', model: 'gpt-4', apiKey: 'k', soulContent: 'Override' })

    await svc.executeStep({
      stepId: 's1',
      instruction: 'test',
      mcpServers: [{ name: 's', url: 'http://s' }],
    })

    expect(svc.isSoulOverrideActive()).toBe(false)
  })
})

describe('POST /execute — not configured', () => {
  it('returns failed when service not configured', async () => {
    const svc = new WorkflowService('test')
    const result = await svc.executeStep({ stepId: 's1', instruction: 'test' })
    expect(result.status).toBe('failed')
    expect(result.error).toContain('not configured')
  })
})

describe('POST /execute — ToolCallRecord', () => {
  it('records serverName, toolName, args, result, durationMs for each tool call', async () => {
    const client = mockMcpClient([{ name: 'add' }], { add: 42 })
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'calc__add', arguments: { a: 1, b: 2 } }] },
      { content: '42' },
    ])
    const svc = createConfiguredService(llm, new Map([['calc', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'add',
      mcpServers: [{ name: 'calc', url: 'http://calc' }],
    })

    expect(result.toolsCalled).toHaveLength(1)
    expect(result.toolsCalled![0].serverName).toBe('calc')
    expect(result.toolsCalled![0].toolName).toBe('add')
    expect(result.toolsCalled![0].durationMs).toBeGreaterThanOrEqual(0)
  })

  it('records multiple tool calls in order of dispatch', async () => {
    const client = mockMcpClient([{ name: 'a' }, { name: 'b' }])
    const llm = mockLlmProvider([
      { content: null, tool_calls: [{ name: 'srv__a', arguments: {} }] },
      { content: null, tool_calls: [{ name: 'srv__b', arguments: {} }] },
      { content: 'done' },
    ])
    const svc = createConfiguredService(llm, new Map([['srv', client]]))

    const result = await svc.executeStep({
      stepId: 's1',
      instruction: 'multi',
      mcpServers: [{ name: 'srv', url: 'http://srv' }],
    })

    expect(result.toolsCalled).toHaveLength(2)
    expect(result.toolsCalled![0].toolName).toBe('a')
    expect(result.toolsCalled![1].toolName).toBe('b')
  })
})