Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* #666 — a complete task with a `kind:'file'` attachment, through the real
* TaskExecutor, tool loop, native registry and safety. Only the model is a
* double: it reads the turn context, calls `clerum__attachment_read` and
* answers with what the tool returned.
*/
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { config as appConfig } from '../../config'
import {
type AttachmentReadLedger,
type AttachmentReadLedgerSnapshot,
attachmentReadBudgets,
} from '../../core/attachments/attachmentReadBudget'
import { ConversationManager } from '../../core/conversation/conversation'
import * as contextManager from '../../core/extensions/contextManager'
import { SimpleEventEmitter } from '../../core/orchestration/eventEmitter'
import { ATTACHED_FILES_INSTRUCTION } from '../../core/orchestration/turnContext'
import { ATTACHMENT_READ_TURN_STOP_MESSAGE } from '../../core/tools/attachmentRead'
import { type ChatMessage, FinishReason, type ToolCall } from '../../core/types'
import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
import type { SingleTurnProvider } from '../../llm/types'
import { McpManager } from '../../mcp/manager'
import type { Task } from '../../queue/types'
import { validateIncomingAttachments } from '../incomingAttachments'
import { TaskExecutor, type TaskExecutorDeps } from '../taskExecutor'
const SENTINEL = 'SENTINEL-666-integration'
const usage = { input_tokens: 1, output_tokens: 1, total_tokens: 2 }
function admittedFile(filename: string, mimeType: string, bytes: Buffer) {
const result = validateIncomingAttachments(
[
{
id: 'file-1',
kind: 'file',
mimeType,
detectedMediaType: mimeType,
encoding: 'base64',
dataBase64: bytes.toString('base64'),
filename,
sizeBytes: bytes.length,
digest: { algorithm: 'sha256', hex: createHash('sha256').update(bytes).digest('hex') },
},
],
{ maxCount: 20, maxBytes: 1_000_000, maxFileBytes: 3_145_728, messageId: 'message-1' }
)
if (!result.ok) throw new Error(`fixture rejected: ${result.error.code}`)
return result.attachments![0]!
}
function lastUserText(messages: ChatMessage[]): string {
const user = [...messages].reverse().find(message => message.role === 'user')
if (!user) throw new Error('the provider received no user message')
return user.content ?? ''
}
interface TaskOptions {
contextWindowTokens?: number
/** The double asks for the next page (or the same offset) on every response. */
readEveryResponse?: boolean
/** The first response asks for an approval-gated shell command instead. */
approvalFirst?: boolean
}
async function runTask(
filename: string,
mimeType: string,
bytes: Buffer,
options: TaskOptions = {}
) {
const attachment = admittedFile(filename, mimeType, bytes)
return { ...(await runTaskWith(attachment, options)), attachment }
}
/** The offset the double asks for next: the page's nextOffset, else the notice's resumeOffset. */
function nextReadOffset(messages: ChatMessage[]): number {
const last = [...messages].reverse().find(message => message.role === 'tool')
if (!last || last.name !== 'clerum__attachment_read') return 0
const payload = toolPayload(last.content ?? '')
if (typeof payload.nextOffset === 'number') return payload.nextOffset
if (typeof payload.resumeOffset === 'number') return payload.resumeOffset
throw new Error(`unexpected attachment read payload kind ${String(payload.kind)}`)
}
/** Without an attachment the double answers directly; with one it reads it first. */
async function runTaskWith(
attachment: ReturnType<typeof admittedFile> | undefined,
options: TaskOptions = {}
) {
const call: ToolCall = {
id: 'read-1',
name: 'clerum__attachment_read',
arguments: { attachmentId: 'file-1' },
}
const providerCalls: Array<{ messages: ChatMessage[]; toolNames: string[] }> = []
let reads = 0
const provider: SingleTurnProvider = {
getProviderType: () => 'openai',
classifyError: () => {
throw new Error('Unexpected provider failure')
},
completeSingleTurn: async () => {
throw new Error('Unexpected non-tool completion')
},
completeSingleTurnWithTools: async (messages, tools) => {
providerCalls.push({
messages: structuredClone(messages),
toolNames: tools.map(tool => tool.name),
})
if (options.approvalFirst && providerCalls.length === 1) {
const shell: ToolCall = {
id: 'shell-1',
name: 'shell_exec',
arguments: { command: "printf 'approved\\n' >> effects.txt" },
}
return { content: null, tool_calls: [shell], usage, finish_reason: FinishReason.ToolUse }
}
if (options.readEveryResponse && attachment) {
reads += 1
const read: ToolCall = {
id: `read-${reads}`,
name: 'clerum__attachment_read',
arguments: { attachmentId: 'file-1', offset: nextReadOffset(messages) },
}
return { content: null, tool_calls: [read], usage, finish_reason: FinishReason.ToolUse }
}
if (providerCalls.length === 1 && attachment) {
return { content: null, tool_calls: [call], usage, finish_reason: FinishReason.ToolUse }
}
const result = messages.find(
message => message.role === 'tool' && message.tool_call_id === call.id
)
return {
content: `The tool returned: ${result?.content ?? 'nothing'}`,
tool_calls: null,
usage,
finish_reason: FinishReason.Stop,
}
},
}
const task: Task = {
id: `attachment-read-${attachment?.filename ?? 'none'}`,
source: 'channel',
status: 'pending',
priority: 'normal',
createdAt: new Date(),
sourceMessage: {
sender: 'authenticated-user',
content: 'Analyze the attached file',
channelType: 'rpc',
channelId: 'isolated-channel',
messageId: 'message-1',
timestamp: new Date().toISOString(),
hostRef: 'fixture-host',
...(attachment ? { attachments: [attachment] } : {}),
},
conversationHistory: [
{ role: 'user', content: 'Analyze the attached file', timestamp: new Date() },
],
responseCallback: vi.fn(async () => {}),
}
const lifecycle = new TaskLifecycle()
lifecycle.register(task)
const coreEvents = new SimpleEventEmitter()
const toolRuns: string[] = []
coreEvents.on('tool:completed', event => {
toolRuns.push(`${String(event.data.toolName)}:${String(event.data.toolCallId)}`)
})
const deps: TaskExecutorDeps = {
conversationManager: new ConversationManager(),
llmProvider: provider,
mcpManager: new McpManager(),
workspaceService: undefined,
modelName: 'test-model',
...(options.contextWindowTokens !== undefined
? { contextWindowTokens: options.contextWindowTokens }
: {}),
approvalConfig: undefined,
config: {
maxTaskDuration: 300000,
maxToolCallsPerTask: options.readEveryResponse ? 100 : 10,
autoStart: true,
taskDelay: 0,
approvalTimeout: 300000,
},
coreEvents,
cronScheduler: null,
taskLifecycle: lifecycle,
onApprovalNeeded: vi.fn(),
onComplete: vi.fn(),
onFail: vi.fn(),
dynamicEnvProvider: () => ({}),
}
const executor = new TaskExecutor(task, deps)
await executor.run()
return { attachment, call, deps, executor, providerCalls, task, toolRuns }
}
function ledgerOf(executor: TaskExecutor): AttachmentReadLedgerSnapshot {
return (
executor as unknown as { attachmentReadLedger: AttachmentReadLedger }
).attachmentReadLedger.snapshot()
}
/** The tool message content is the `<tool_output>` wrapper around the page JSON. */
function toolPayload(content: string): Record<string, unknown> {
const bodyStart = content.indexOf('\n')
const bodyEnd = content.lastIndexOf('\n</tool_output>')
if (bodyStart < 0 || bodyEnd < 0) throw new Error('tool message is not wrapped')
return JSON.parse(content.slice(bodyStart + 1, bodyEnd)) as Record<string, unknown>
}
describe('clerum__attachment_read through a complete task (#666)', () => {
const saved = {
enableApproval: appConfig.enableApproval,
dynamicToolsEnabled: appConfig.dynamicToolsEnabled,
promptCacheEnabled: appConfig.promptCacheEnabled,
}
afterEach(() => {
Object.assign(appConfig, saved)
vi.restoreAllMocks()
})
it('lists the text file, reads it in the same turn and answers with its sentinel', async () => {
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: true,
})
const { attachment, call, deps, executor, providerCalls } = await runTask(
'notes.txt',
'text/plain',
Buffer.from(`Quarterly notes. ${SENTINEL}\n`)
)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(providerCalls).toHaveLength(2)
// Turn 1: the tool is presented and the file is listed, never inlined.
const reference = attachment.fileReference!
const firstUserText = lastUserText(providerCalls[0]!.messages)
expect(providerCalls[0]!.toolNames).toContain('clerum__attachment_read')
expect(firstUserText).toContain(
`attached_file: id="file-1" name="notes.txt" class=${reference.class} bytes=${reference.byteLength} reader=text\n`
)
expect(firstUserText).toContain(ATTACHED_FILES_INSTRUCTION)
expect(firstUserText).not.toContain(SENTINEL)
// Turn 2: the tool trace carries the call and its sanitized text result.
const second = providerCalls[1]!.messages
expect(second.flatMap(message => message.tool_calls ?? []).map(tool => tool.name)).toContain(
'clerum__attachment_read'
)
const result = second.find(
message => message.role === 'tool' && message.tool_call_id === call.id
)
expect(result?.content).toContain(SENTINEL)
expect(result?.content).toContain('"kind":"text"')
expect(result?.content).not.toContain('�')
expect(deps.onComplete).toHaveBeenCalledTimes(1)
})
it('bounds the emitted page with the explicit turn window and debits the shared ledger', async () => {
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: true,
})
// window 25_000 → page 2_500 / turn 7_500 with the notice reserve covered,
// so the page binds (limit=page_budget) and a 65 KiB dense file cannot be
// emitted whole; under the default 100k window it would arrive complete.
const fileBytes = 65_535
const { call, deps, executor, providerCalls } = await runTask(
'dense.txt',
'text/plain',
Buffer.from('A'.repeat(fileBytes)),
{ contextWindowTokens: 25_000 }
)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(providerCalls).toHaveLength(2)
const result = providerCalls[1]!.messages.find(
message => message.role === 'tool' && message.tool_call_id === call.id
)
expect(result?.content).toBeTruthy()
const payload = toolPayload(result!.content)
expect(payload.kind).toBe('text')
expect(payload.truncated).toBe(true)
expect(payload.limit).toBe('page_budget')
const byteRange = payload.byteRange as { offset: number; length: number }
expect(byteRange.offset).toBe(0)
expect(byteRange.length).toBeGreaterThan(0)
expect(byteRange.length).toBeLessThan(fileBytes)
expect(typeof payload.nextOffset).toBe('number')
expect(payload.nextOffset as number).toBeGreaterThan(0)
expect(deps.onComplete).toHaveBeenCalledTimes(1)
})
it('returns a binary result for a PDF, and the model never sees its bytes', async () => {
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: true,
})
const { attachment, call, deps, executor, providerCalls } = await runTask(
'report.pdf',
'application/pdf',
Buffer.from(`%PDF-1.7\n${SENTINEL}\n%%EOF\n`)
)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(providerCalls).toHaveLength(2)
expect(lastUserText(providerCalls[0]!.messages)).toContain(
`attached_file: id="file-1" name="report.pdf" class=${attachment.fileReference!.class} bytes=${attachment.fileReference!.byteLength} reader=none\n`
)
const result = providerCalls[1]!.messages.find(
message => message.role === 'tool' && message.tool_call_id === call.id
)
// Witness: the tool answered with the typed binary result.
expect(result?.content).toContain('"kind":"binary"')
expect(result?.content).toContain('"reason":"no_reader_for_class"')
const everything = JSON.stringify(providerCalls.map(entry => entry.messages))
expect(everything).not.toContain(SENTINEL)
})
it('lists attached files with the prompt cache off, so the tool stays usable', async () => {
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: false,
})
const { attachment, call, deps, executor, providerCalls } = await runTask(
'notes.txt',
'text/plain',
Buffer.from(`Quarterly notes. ${SENTINEL}\n`)
)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
const firstUserText = lastUserText(providerCalls[0]!.messages)
expect(firstUserText.startsWith('<turn-context>')).toBe(true)
expect(firstUserText).toContain(
`attached_file: id="file-1" name="notes.txt" class=${attachment.fileReference!.class} bytes=${attachment.fileReference!.byteLength} reader=text\n`
)
const result = providerCalls[1]!.messages.find(
message => message.role === 'tool' && message.tool_call_id === call.id
)
expect(result?.content).toContain(SENTINEL)
})
it('adds no turn-context block with the prompt cache off and no attached file', async () => {
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: false,
})
const { deps, executor, providerCalls } = await runTaskWith(undefined)
// Witness: the task ran and the model received the user's message.
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(providerCalls).toHaveLength(1)
expect(lastUserText(providerCalls[0]!.messages)).toBe('Analyze the attached file')
expect(providerCalls[0]!.toolNames).not.toContain('clerum__attachment_read')
})
describe('notice overdraft ceiling (A15 U1)', () => {
const WINDOW = 8_192
const { turnTokens } = attachmentReadBudgets(WINDOW)
const ceiling = turnTokens + Math.floor(turnTokens / 2)
const reads = (from: number, to: number): string[] =>
Array.from({ length: to - from + 1 }, (_, i) => `clerum__attachment_read:read-${from + i}`)
/** The kind of read-k's result as the model received it on the following call. */
const kindSeenAfter = (
providerCalls: Array<{ messages: ChatMessage[] }>,
index: number,
k: number
) => {
const message = providerCalls[index]!.messages.find(
entry => entry.role === 'tool' && entry.tool_call_id === `read-${k}`
)
if (!message) throw new Error(`the model never received read-${k}`)
return toolPayload(message.content ?? '').kind
}
it('ends the task with a stop message instead of failing it when the model keeps reading', async () => {
const measurement = vi.spyOn(contextManager, 'toolMessageBudgetTokens')
Object.assign(appConfig, {
enableApproval: false,
dynamicToolsEnabled: false,
promptCacheEnabled: true,
})
const { deps, executor, providerCalls, task, toolRuns } = await runTask(
'long.txt',
'text/plain',
Buffer.alloc(400_000, 'q'),
{ contextWindowTokens: WINDOW, readEveryResponse: true }
)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(deps.onComplete).toHaveBeenCalledTimes(1)
expect(vi.mocked(task.responseCallback!).mock.calls.map(([arg]) => arg.response)).toEqual([
ATTACHMENT_READ_TURN_STOP_MESSAGE,
])
// Measured: 19 reads ran and the model was called once per read before
// the stop, never after it (two pages, then 16 charged notices).
expect(toolRuns).toEqual(reads(1, 19))
expect(providerCalls).toHaveLength(19)
// Witness: reads 1-18 reached the model as a page or a plain notice.
const kinds = Array.from({ length: 18 }, (_, i) => kindSeenAfter(providerCalls, i + 1, i + 1))
expect(kinds.slice(0, 2)).toEqual(['text', 'text'])
expect(kinds.slice(2).every(kind => kind === 'read_budget_exhausted')).toBe(true)
// After the first exhausted reply, retries must not keep fitting full
// text candidates that cannot reach the model. Keep the real measurer:
// this exercises TaskExecutor's budget fence, not a cheaper test counter.
const retryMeasurements = measurement.mock.calls
.map(([message]) => message)
.filter(message => Number(message.tool_call_id?.slice('read-'.length)) >= 4)
expect(retryMeasurements.length).toBeGreaterThan(0)
expect(
retryMeasurements.some(message => toolPayload(message.content ?? '').kind === 'text')
).toBe(false)
expect(ledgerOf(executor).spentTokens).toBeGreaterThan(turnTokens)
expect(ledgerOf(executor).spentTokens).toBeLessThanOrEqual(ceiling)
})
it('ends the task the same way after a resume that cannot prove the earlier spend', async () => {
const dir = await mkdtemp(join(tmpdir(), 'a15-turn-stop-'))
const savedWorkspace = appConfig.nativeTool.workspacePath
try {
Object.assign(appConfig, {
enableApproval: true,
dynamicToolsEnabled: false,
promptCacheEnabled: true,
})
appConfig.nativeTool.workspacePath = dir
const { deps, executor, providerCalls, task, toolRuns } = await runTask(
'long.txt',
'text/plain',
Buffer.alloc(400_000, 'q'),
{ contextWindowTokens: WINDOW, readEveryResponse: true, approvalFirst: true }
)
expect(executor.executorState).toBe('waiting_approval')
const approval = executor.pendingApproval!
expect(approval.task_budget?.attachmentReadLedger).toEqual({
reads: 0,
spentTokens: 0,
bytesRead: 0,
})
// A missing carrier makes the resume exhaust the ledger (fail closed).
delete approval.task_budget!.attachmentReadLedger
await executor.resumeAfterApproval(false)
expect(deps.onFail).not.toHaveBeenCalled()
expect(executor.executorState).toBe('completed')
expect(deps.onComplete).toHaveBeenCalledTimes(1)
expect(vi.mocked(task.responseCallback!).mock.calls.map(([arg]) => arg.response)).toEqual([
ATTACHMENT_READ_TURN_STOP_MESSAGE,
])
// Measured: the exhausted ledger pays 8 notices from the overdraft and
// the 9th read stops the turn.
expect(toolRuns).toEqual(['shell_exec:shell-1', ...reads(1, 9)])
// The shell call, then one model call after the shell result and after
// each of reads 1-8.
expect(providerCalls).toHaveLength(10)
// Witness: reads 1-8 reached the model as notices, never as text.
for (let k = 1; k <= 8; k++) {
expect(kindSeenAfter(providerCalls, k + 1, k)).toBe('read_budget_exhausted')
}
const ledger = ledgerOf(executor)
expect(ledger.reads).toBe(32)
expect(ledger.bytesRead).toBe(0)
expect(ledger.spentTokens).toBeGreaterThan(turnTokens)
expect(ledger.spentTokens).toBeLessThanOrEqual(ceiling)
// The stop is the ceiling's: the room left is less than one notice.
const noticeCost = (ledger.spentTokens - turnTokens) / 8
expect(Number.isInteger(noticeCost)).toBe(true)
expect(ceiling - ledger.spentTokens).toBeLessThan(noticeCost)
} finally {
appConfig.nativeTool.workspacePath = savedWorkspace
await rm(dir, { recursive: true, force: true })
}
})
})
})