Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
6.7 KB
Raw Blame
import { afterEach, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { config as appConfig } from '../../config'
import { ConversationManager } from '../../core/conversation/conversation'
import { LlmError, LlmErrorCode } from '../../core/errors'
import { toolMessageBudgetTokens } from '../../core/extensions/contextManager'
import { SimpleEventEmitter } from '../../core/orchestration/eventEmitter'
import type { TokenCounter } from '../../core/tokenizer/tokenCounter'
import { type ChatMessage, FinishReason } from '../../core/types'
import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
import { FailoverEngine } from '../../llm/failover/engine'
import type { LlmPolicy } from '../../llm/failover/types'
import type { SingleTurnProvider } from '../../llm/types'
import { McpManager } from '../../mcp/manager'
import type { Task } from '../../queue/types'
import { validateIncomingAttachments } from '../incomingAttachments'
import { TaskExecutor } from '../taskExecutor'

const saved = {
  enableApproval: appConfig.enableApproval,
  dynamicToolsEnabled: appConfig.dynamicToolsEnabled,
  tokenizerDryrun: appConfig.tokenizerDryrun,
}
afterEach(() => Object.assign(appConfig, saved))

it.each([true, false])(
  'bounds the page for a smaller fallback tokenizer (fail before page: %s)',
  async failBeforePage => {
    Object.assign(appConfig, {
      enableApproval: false,
      dynamicToolsEnabled: false,
      tokenizerDryrun: true,
    })
    const bytes = Buffer.from('A'.repeat(65_535))
    const admitted = validateIncomingAttachments(
      [
        {
          id: 'file-1',
          kind: 'file',
          filename: 'public-dense.txt',
          mimeType: 'text/plain',
          detectedMediaType: 'text/plain',
          encoding: 'base64',
          sizeBytes: bytes.length,
          dataBase64: bytes.toString('base64'),
          digest: { algorithm: 'sha256', hex: createHash('sha256').update(bytes).digest('hex') },
        },
      ],
      { maxCount: 20, maxBytes: 1_000_000, maxFileBytes: 11_534_336, messageId: 'window-message' }
    )
    if (!admitted.ok) throw new Error('Invalid public file fixture')
    const usage = { input_tokens: 1, output_tokens: 1, total_tokens: 2 }
    const received: ChatMessage[][] = []
    let primaryCalls = 0
    const readResponse = () => ({
      content: null,
      usage,
      finish_reason: FinishReason.ToolUse,
      tool_calls: [
        {
          id: 'window-read',
          name: 'clerum__attachment_read',
          arguments: { attachmentId: 'file-1' },
        },
      ],
    })
    const unexpected = () => {
      throw new Error('Unexpected completion')
    }
    const primary: SingleTurnProvider = {
      getProviderType: () => 'openai',
      classifyError: unexpected,
      completeSingleTurn: unexpected,
      completeSingleTurnWithTools: vi.fn(async () => {
        primaryCalls += 1
        if (!failBeforePage && primaryCalls === 1) return readResponse()
        throw new LlmError('Public quota fixture', 'openai', LlmErrorCode.InsufficientQuota, false)
      }),
    }
    const fallback: SingleTurnProvider = {
      getProviderType: () => 'claude',
      classifyError: unexpected,
      completeSingleTurn: unexpected,
      completeSingleTurnWithTools: async messages => {
        received.push(structuredClone(messages))
        if (failBeforePage && received.length === 1) return readResponse()
        return {
          content: 'Page received',
          tool_calls: null,
          usage,
          finish_reason: FinishReason.Stop,
        }
      },
    }
    const fallbackCounter: TokenCounter = {
      providerName: 'claude',
      modelName: 'public-small-model',
      warmup: vi.fn(async () => {}),
      countSync: messages =>
        Math.ceil(messages.reduce((n, m) => n + (m.content?.length ?? 0), 0) / 2) + 6,
      count: vi.fn(async () => {
        throw new Error('Network counting is forbidden in page measurement')
      }),
      recordObservedUsage: () => {},
      lastObservedInputTokens: () => null,
    }
    Object.assign(fallback, { createTokenCounter: () => fallbackCounter })
    const policy: LlmPolicy = {
      cooldownSeconds: 300,
      triggerOn: ['insufficient_quota'],
      fallbacks: [{ provider: 'claude', model: 'public-small-model' }],
    }
    const catalog = vi.fn((provider: string, model: string) =>
      provider === 'claude' && model === 'public-small-model' ? 32_000 : undefined
    )
    const task: Task = {
      id: 'public-window-task',
      source: 'channel',
      status: 'pending',
      priority: 'normal',
      createdAt: new Date(),
      responseCallback: vi.fn(async () => {}),
      sourceMessage: {
        sender: 'authenticated-user',
        content: 'Read the file',
        channelType: 'rpc',
        channelId: 'isolated-channel',
        messageId: 'window-message',
        hostRef: 'fixture-host',
        timestamp: new Date().toISOString(),
        attachments: admitted.attachments,
      },
      conversationHistory: [{ role: 'user', content: 'Read the file', timestamp: new Date() }],
    }
    const lifecycle = new TaskLifecycle()
    lifecycle.register(task)
    const onFail = vi.fn()
    const executor = new TaskExecutor(task, {
      conversationManager: new ConversationManager(),
      llmProvider: primary,
      modelName: 'public-primary-model',
      contextWindowTokens: 128_000,
      mcpManager: new McpManager(),
      workspaceService: undefined,
      approvalConfig: undefined,
      config: {
        maxTaskDuration: 300_000,
        maxToolCallsPerTask: 10,
        autoStart: true,
        taskDelay: 0,
        approvalTimeout: 300_000,
      },
      coreEvents: new SimpleEventEmitter(),
      cronScheduler: null,
      taskLifecycle: lifecycle,
      onApprovalNeeded: vi.fn(),
      onComplete: vi.fn(),
      onFail,
      failover: {
        engine: new FailoverEngine(policy),
        policy,
        buildProvider: () => fallback,
        contextWindowForPair: catalog,
      },
    })
    await executor.run()
    expect(onFail).not.toHaveBeenCalled()
    expect(executor.executorState).toBe('completed')
    expect(primary.completeSingleTurnWithTools).toHaveBeenCalledTimes(failBeforePage ? 1 : 2)
    expect(received).toHaveLength(failBeforePage ? 2 : 1)
    const page = received
      .at(-1)!
      .find(message => message.role === 'tool' && message.tool_call_id === 'window-read')
    expect(page?.content).toContain('"kind":"text"')
    expect(page?.content).toContain('"truncated":true')
    expect(toolMessageBudgetTokens(page!, undefined, true)).toBeLessThanOrEqual(3_200)
    expect(fallbackCounter.countSync([page!])).toBeLessThanOrEqual(3_200)
    expect(fallbackCounter.count).not.toHaveBeenCalled()
    expect(catalog).toHaveBeenCalledWith('claude', 'public-small-model')
  }
)