Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
13.8 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest'
import { BudgetClient } from '../../budget/budgetClient'
import type { BudgetCheckRequest } from '../../budget/types'
import { config as appConfig } from '../../config'
import { LlmErrorCode } from '../../core/errors'
import type { SingleTurnProvider } from '../../llm'
import { logger } from '../../logger'
import type { McpClientConnection, McpClientFactory } from '../stepRouter'
import type { McpHostRuntimeAuth } from '../userApprovalRequester'
import { WorkflowService } from '../workflowService'

// Canonical workflow run id (UUID prefix) so usage-reporting validation passes
// when a UsageReporter is wired; budget tests below omit the reporter, so this
// is only used to populate task_ref / attribution.
const workflowExecutionId =
  '11111111-1111-4111-8111-111111111111:my-recipe:2026-05-09T00:00:00.000Z'
const workflowTeamId = '22222222-2222-4222-8222-222222222222'
const workflowUserId = '33333333-3333-4333-8333-333333333333'

function mockRuntimeAuth(
  recipeName: string,
  recipeNamespace = 'sandbox-recipes',
  hostRef = `${recipeNamespace}/${recipeName}`
): McpHostRuntimeAuth {
  return {
    accessToken: 'a',
    refreshToken: 'r',
    baseUrl: 'http://gw',
    hostRef,
    recipeName,
    recipeNamespace,
  }
}

function mockLlmProvider(content = 'final answer'): SingleTurnProvider {
  return {
    completeSingleTurn: vi.fn().mockResolvedValue({
      content,
      usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
      finish_reason: 'stop',
    }),
    completeSingleTurnWithTools: vi.fn().mockResolvedValue({
      content,
      tool_calls: null,
      usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
      finish_reason: 'stop',
    }),
    getProviderType: () => 'openai',
    classifyError: vi.fn(() => ({
      code: LlmErrorCode.ApiCallFailed,
      retryable: true,
      message: 'mock',
    })),
  }
}

function mockMcpClient(tools: Array<{ name: string }> = []): McpClientConnection {
  return {
    connect: vi.fn().mockResolvedValue(undefined),
    listTools: vi.fn().mockResolvedValue(tools.map(t => ({ name: t.name, description: '' }))),
    callTool: vi
      .fn()
      .mockImplementation((name: string) =>
        Promise.resolve({ content: `result-${name}`, isError: false })
      ),
    disconnect: vi.fn().mockResolvedValue(undefined),
  }
}

function mockMcpFactory(): McpClientFactory {
  return () => mockMcpClient()
}

interface FetchTrace {
  checkBodies: BudgetCheckRequest[]
  releaseBodies: Array<Record<string, unknown>>
  fetchImpl: typeof fetch
}

/**
 * Routes /budgets/check and /budgets/release. `verdict` is the JSON returned for
 * a check; pass `'throw'` to simulate a transport failure (fail-open path).
 */
function makeBudgetFetch(verdict: Record<string, unknown> | 'throw'): FetchTrace {
  const checkBodies: BudgetCheckRequest[] = []
  const releaseBodies: Array<Record<string, unknown>> = []
  const fetchImpl = vi.fn().mockImplementation((url: string, init: { body: string }) => {
    if (url.endsWith('/budgets/check')) {
      checkBodies.push(JSON.parse(init.body) as BudgetCheckRequest)
      if (verdict === 'throw') return Promise.reject(new Error('network down'))
      return Promise.resolve({ ok: true, status: 200, json: async () => verdict })
    }
    if (url.endsWith('/budgets/release')) {
      releaseBodies.push(JSON.parse(init.body) as Record<string, unknown>)
      return Promise.resolve({ ok: true, status: 200, json: async () => ({ released: 1 }) })
    }
    return Promise.reject(new Error(`unexpected url: ${url}`))
  })
  return { checkBodies, releaseBodies, fetchImpl: fetchImpl as unknown as typeof fetch }
}

function makeService(
  budgetClient: BudgetClient | null,
  llm = mockLlmProvider()
): { svc: WorkflowService; llm: SingleTurnProvider } {
  const svc = new WorkflowService('my-recipe-12345', {
    llmFactory: () => llm,
    mcpClientFactory: mockMcpFactory(),
    runtimeAuth: mockRuntimeAuth('my-recipe'),
    budgetClient,
  })
  svc.configure({
    provider: 'openai',
    model: 'gpt-4o',
    apiKey: 'sk-123',
    llmSecretName: 'chatllm-api-keys',
  })
  return { svc, llm }
}

describe('WorkflowService — P3 pre-step budget check', () => {
  afterEach(() => {
    vi.restoreAllMocks()
    appConfig.budgetsEnabled = false
  })

  it('flag off: no-op, never calls the check, step executes', async () => {
    appConfig.budgetsEnabled = false
    const trace = makeBudgetFetch({ allowed: true })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const { svc, llm } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('completed')
    expect(trace.checkBodies).toHaveLength(0)
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
  })

  it('flag on + allowed: executes and sends source_kind=workflow + recipe_name + attribution', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: true })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const { svc, llm } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('completed')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
    expect(trace.checkBodies).toHaveLength(1)
    expect(trace.checkBodies[0]).toMatchObject({
      host_ref: 'sandbox-recipes/my-recipe',
      context_ref: null,
      llm_secret_name: 'chatllm-api-keys',
      provider: 'openai',
      model: 'gpt-4o',
      source_kind: 'workflow',
      recipe_name: 'my-recipe',
      user_id: workflowUserId,
      team_id: workflowTeamId,
      cron_job_id: null,
    })
    expect(trace.checkBodies[0]?.task_ref).toContain('s1')
  })

  it('flag on + deny: step fails with budget_exceeded and the LLM is never called', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: false, reason: 'global_tokens_exceeded' })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const { svc, llm } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('failed')
    expect(res.error).toContain('budget_exceeded')
    expect(res.error).toContain('global_tokens_exceeded')
    expect(trace.checkBodies).toHaveLength(1)
    expect(llm.completeSingleTurn).not.toHaveBeenCalled()
    expect(llm.completeSingleTurnWithTools).not.toHaveBeenCalled()
  })

  it('deny log keeps the user-controlled stepId and reason out of the message string', async () => {
    appConfig.budgetsEnabled = true
    const reason = 'global_tokens_exceeded\nforged-line level=info'
    const trace = makeBudgetFetch({ allowed: false, reason })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined)
    const { svc } = makeService(client)
    const stepId = 's1\n[WorkflowService] forged'

    const res = await svc.executeStep({
      stepId,
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('failed')
    expect(warn).toHaveBeenCalledWith(
      { component: 'WorkflowService', stepId, reason },
      'Step denied by budget'
    )
  })

  it('unpriced usage: warns budget_unpriced_usage with the pairs and still executes the step', async () => {
    appConfig.budgetsEnabled = true
    const unpriced = [
      { provider: 'openai', model: 'gpt-4o' },
      { provider: 'anthropic', model: 'claude-3-5-sonnet' },
    ]
    const trace = makeBudgetFetch({ allowed: true, unpriced })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined)
    const { svc, llm } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('completed')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
    expect(warn).toHaveBeenCalledWith(
      expect.objectContaining({
        component: 'WorkflowService',
        stepId: 's1',
        recipe_name: 'my-recipe',
        pairs: unpriced,
      }),
      'budget_unpriced_usage'
    )
  })

  it('no unpriced field: does not warn budget_unpriced_usage', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: true })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const warn = vi.spyOn(logger, 'warn').mockImplementation(() => undefined)
    const { svc } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('completed')
    expect(warn).not.toHaveBeenCalledWith(expect.anything(), 'budget_unpriced_usage')
  })

  it('fail-open: a broken check lets the step proceed', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch('throw')
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
    const { svc, llm } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })

    expect(res.status).toBe('completed')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
    expect(warn).toHaveBeenCalled()
  })

  it('budgetClient=null: explicit disable is a no-op even with the flag on', async () => {
    appConfig.budgetsEnabled = true
    const { svc, llm } = makeService(null)
    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })
    expect(res.status).toBe('completed')
    expect(llm.completeSingleTurn).toHaveBeenCalledTimes(1)
  })

  it('P2b: releases a danger-zone reservation on step terminal via task_ref', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: true, reservationIds: ['res-1'] })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const { svc } = makeService(client)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })
    // release() is fire-and-forget (not awaited on the step path); flush microtasks.
    await new Promise(resolve => setImmediate(resolve))

    expect(res.status).toBe('completed')
    expect(trace.releaseBodies).toHaveLength(1)
    expect(trace.releaseBodies[0]?.task_ref).toBe(trace.checkBodies[0]?.task_ref)
    // host_ref MUST match the check's (== JWT hostRefs[0]) so control-api binds
    // and host-scopes the drop to the reservation this step created.
    expect(trace.releaseBodies[0]?.host_ref).toBe(trace.checkBodies[0]?.host_ref)
    expect(trace.releaseBodies[0]?.host_ref).toBe('sandbox-recipes/my-recipe')
  })

  it('P2b: still releases the reservation when the step ends in FAILED', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: true, reservationIds: ['res-1'] })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const llm = mockLlmProvider()
    const fail = new Error('llm down')
    ;(llm.completeSingleTurn as ReturnType<typeof vi.fn>).mockRejectedValue(fail)
    ;(llm.completeSingleTurnWithTools as ReturnType<typeof vi.fn>).mockRejectedValue(fail)
    ;(llm.classifyError as ReturnType<typeof vi.fn>).mockReturnValue({
      code: LlmErrorCode.ApiCallFailed,
      retryable: false,
      message: 'llm down',
    })
    const { svc } = makeService(client, llm)

    const res = await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })
    await new Promise(resolve => setImmediate(resolve))

    expect(res.status).toBe('failed')
    // The reservation must be freed on ANY terminal exit, not just completion.
    expect(trace.releaseBodies).toHaveLength(1)
    expect(trace.releaseBodies[0]?.task_ref).toBe(trace.checkBodies[0]?.task_ref)
  })

  it('no reservation: nothing is released on terminal', async () => {
    appConfig.budgetsEnabled = true
    const trace = makeBudgetFetch({ allowed: true })
    const client = new BudgetClient({
      baseUrl: 'http://gw',
      getAccessToken: () => 't',
      fetchImpl: trace.fetchImpl,
    })
    const { svc } = makeService(client)

    await svc.executeStep({
      stepId: 's1',
      instruction: 'hello',
      contextVars: { workflowExecutionId, workflowTeamId, workflowUserId },
    })
    await new Promise(resolve => setImmediate(resolve))

    expect(trace.releaseBodies).toHaveLength(0)
  })
})