Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { spawn } from 'child_process'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { loadDevStore } from '../../__tests__/fixtures/devGfsStoreFixture'
import { config as appConfig } from '../../config'
import { ConversationManager } from '../../core/conversation/conversation'
import { SimpleEventEmitter } from '../../core/orchestration/eventEmitter'
import { ShellTool } from '../../core/tools/shell'
import { type ChatMessage, FinishReason, type ToolCall } from '../../core/types'
import { GfsDownloadStore, GfsDownloadStoreError } from '../../internalTools/gfsDownloadStore'
import { TaskLifecycle } from '../../lifecycle/taskLifecycle'
import type { SingleTurnProvider } from '../../llm/types'
import { logger } from '../../logger'
import type { Task, TaskSource } from '../../queue/types'
import { resolveCallerRootBinding } from '../../workspace/callerRootBinding'
import { ScopedWorkspaceProvider } from '../../workspace/scopedWorkspace'
import { TaskExecutor, type TaskExecutorDeps } from '../taskExecutor'
vi.mock('child_process', async importOriginal => {
const actual = await importOriginal<typeof import('child_process')>()
return { ...actual, spawn: vi.fn(actual.spawn) }
})
const { clientFactory } = vi.hoisted(() => ({ clientFactory: vi.fn() }))
// Only GFSC's external boundary is doubled; the store, registry, approval gate,
// TaskExecutor and shell are real.
vi.mock('../../internalTools/gfsClient', async importOriginal => ({
...(await importOriginal<typeof import('../../internalTools/gfsClient')>()),
getGfsToolScopes: () => new Set(['gfs.read']),
createGfscClient: clientFactory,
}))
const largeResourceId = 'c'.repeat(32)
const largeFile = {
source: {
kind: 'gfs' as const,
drive: 'main',
resourceId: largeResourceId,
gfsUri: `gfs://main/${largeResourceId}`,
name: 'large.csv',
version: 7,
},
size: 3_836_961,
}
let gfsClient: {
accessible: ReturnType<typeof vi.fn>
list: ReturnType<typeof vi.fn>
read: ReturnType<typeof vi.fn>
readMetadata: ReturnType<typeof vi.fn>
download: ReturnType<typeof vi.fn>
stat: ReturnType<typeof vi.fn>
resolve: ReturnType<typeof vi.fn>
}
const savedConfig = {
enableApproval: appConfig.enableApproval,
codexToolPresentation: appConfig.codexToolPresentation,
dynamicToolsEnabled: appConfig.dynamicToolsEnabled,
promptCacheEnabled: appConfig.promptCacheEnabled,
}
const savedNativeTool = { ...appConfig.nativeTool }
const roots: string[] = []
const stores: GfsDownloadStore[] = []
/** Spies installed with vi.spyOn by a test; restored before the stores close. */
const installedSpies: Array<{ mockRestore: () => void }> = []
beforeEach(() => {
Object.assign(appConfig, {
enableApproval: true,
codexToolPresentation: 'direct',
dynamicToolsEnabled: false,
promptCacheEnabled: false,
})
vi.clearAllMocks()
const unexpected = (name: string) =>
vi.fn(async () => {
throw new Error(`Unexpected GFSC ${name} call`)
})
gfsClient = {
accessible: unexpected('accessible'),
list: unexpected('list'),
read: unexpected('read'),
readMetadata: vi.fn(async () => structuredClone(largeFile)),
download: unexpected('download'),
stat: unexpected('stat'),
resolve: unexpected('resolve'),
}
clientFactory.mockReturnValue(gfsClient)
})
afterEach(async () => {
for (const spy of installedSpies.splice(0)) spy.mockRestore()
for (const store of stores.splice(0)) await store.close().catch(() => undefined)
for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true })
Object.assign(appConfig, savedConfig)
Object.assign(appConfig.nativeTool, savedNativeTool)
})
/**
* A store whose Host root is a symlink: initialize refuses it with
* workspace_unavailable. The shell workspace stays on the real temp root.
*/
async function unavailableStore() {
const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-'))
roots.push(root)
const target = join(root, 'store-target')
await fs.mkdir(target, { mode: 0o700 })
const link = join(root, 'store-link')
await fs.symlink(target, link)
const store = new GfsDownloadStore(link)
stores.push(store)
// Capture the rejection instead of swallowing it so tests can assert which
// failure the store reported; an initialize that resolves breaks the fixture
// loudly.
const initError = await store.initialize().then(
() => {
throw new Error('unavailableStore fixture: initialize() resolved on a symlinked Host root')
},
(error: unknown) => error
)
expect(initError).toMatchObject({ code: 'workspace_unavailable' })
return { root, store, initError }
}
/**
* A store whose Host root holds what the pre-#1028 image left on disk, as its
* real store wrote it (devGfsStoreFixture, `restarted`): a ledger with every
* record quarantined, an expired and a live processing lease of a killed
* writer, the v2 writer fence and its database. The dev store itself was
* unavailable on this state (#1019). `live` picks which of the two leases the
* caller checks.
*/
async function preRedesignStore({ live = false }: { live?: boolean } = {}) {
const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-legacy-'))
roots.push(root)
const devStore = loadDevStore(root, 'restarted')
expect(devStore.oldStoreAvailable).toBe(false)
expect(Date.parse(devStore.leases.expired.expiresAt)).toBeLessThan(Date.now())
expect(Date.parse(devStore.leases.live.expiresAt)).toBeGreaterThan(Date.now())
const seededLease = live ? devStore.leases.live : devStore.leases.expired
// Witness for every X1 case: the old store is on disk before the start and
// the start retires it, so availability cannot come from a missing fixture.
await expect(fs.lstat(devStore.storeRoot)).resolves.toBeDefined()
const warn = vi.spyOn(logger, 'warn')
const store = new GfsDownloadStore(root)
stores.push(store)
await store.initialize()
expect(warn).toHaveBeenCalledWith(
expect.objectContaining({ component: 'GfsDownloadStore' }),
'Retired the pre-#1028 GFS download store'
)
warn.mockRestore()
await expect(fs.lstat(join(root, '.gfs-download-store'))).rejects.toMatchObject({
code: 'ENOENT',
})
return { root, store, seededLease }
}
type StoreFixture = { root: string; store: GfsDownloadStore; initError?: unknown }
async function healthyStore() {
const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-healthy-'))
roots.push(root)
const store = new GfsDownloadStore(root)
stores.push(store)
await store.initialize()
return { root, store }
}
/** Hooks called by the scripted provider at the start and end of every model turn. */
type TurnObserver = {
turnStarted(turn: number): void
turnReturned(turn: number, toolCall: ToolCall | undefined): void
}
async function shellScenario({
source,
approvalEnabled = true,
shellApprovalDisabled = false,
healthy = false,
preRedesign = false,
channelCaller = false,
missingCallerRoot = false,
calls = [{ id: 'shell-safety', name: 'shell_exec', arguments: { command: 'pwd' } }],
turnObserver,
dynamicEnv = {},
storeFixture,
}: {
source: TaskSource
approvalEnabled?: boolean
shellApprovalDisabled?: boolean
healthy?: boolean
preRedesign?: boolean
channelCaller?: boolean
missingCallerRoot?: boolean
calls?: ToolCall[]
turnObserver?: TurnObserver
/** Operator-managed env the shell receives through `dynamicEnvProvider`. */
dynamicEnv?: Record<string, string>
/** Builds the store the executor is wired to; overrides the flags above. */
storeFixture?: () => Promise<StoreFixture>
}) {
appConfig.enableApproval = approvalEnabled
const fixture: StoreFixture = storeFixture
? await storeFixture()
: preRedesign
? await preRedesignStore()
: healthy
? await healthyStore()
: await unavailableStore()
const { root, store } = fixture
const storeInitError = fixture.initError
const sourceMessage = channelCaller
? {
sender: 'gfs-safety-caller',
content: 'run pwd',
channelType: 'rpc' as const,
channelId: 'gfs-safety-channel',
messageId: 'gfs-safety-message',
timestamp: new Date().toISOString(),
hostRef: 'gfs-safety-host',
}
: undefined
let callerWorkspace = join(root, 'users', '_system')
if (sourceMessage) {
const binding = resolveCallerRootBinding(new ScopedWorkspaceProvider(root), sourceMessage)
if (!binding.root) throw new Error(`Test caller binding failed: ${binding.failureCode}`)
callerWorkspace = binding.root
}
await fs.mkdir(callerWorkspace, { recursive: true, mode: 0o700 })
const providerCalls: ChatMessage[][] = []
const advertisedTools: string[][] = []
const provider: SingleTurnProvider = {
getProviderType: () => 'codex-subscription',
classifyError: () => ({
code: 'LLM_UNKNOWN_ERROR' as never,
retryable: false,
message: 'fixture classification is never used',
}),
completeSingleTurn: async () => {
throw new Error('Unexpected non-tool completion')
},
completeSingleTurnWithTools: async (messages, tools) => {
const turn = providerCalls.length
turnObserver?.turnStarted(turn)
providerCalls.push(structuredClone(messages))
advertisedTools.push(tools.map(tool => tool.name))
const call = calls[turn]
turnObserver?.turnReturned(turn, call)
return {
content: call ? null : 'Shell finished.',
tool_calls: call ? [call] : null,
usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
finish_reason: call ? FinishReason.ToolUse : FinishReason.Stop,
}
},
}
const task: Task = {
id: '22222222-2222-4222-8222-222222222222',
source,
status: 'pending',
priority: 'normal',
createdAt: new Date(),
...(sourceMessage ? { sourceMessage } : {}),
conversationHistory: [{ role: 'user', content: 'run pwd', timestamp: new Date() }],
responseCallback: vi.fn(async () => undefined),
}
const lifecycle = new TaskLifecycle()
lifecycle.register(task)
const onFail = vi.fn()
const onApprovalNeeded = vi.fn()
const deps: TaskExecutorDeps = {
conversationManager: new ConversationManager(),
llmProvider: provider,
mcpManager: null,
workspaceService: undefined,
gfsDownloadStore: store,
gfsCallerWorkspacePath: missingCallerRoot ? undefined : callerWorkspace,
modelName: 'fixture-model',
approvalConfig: {
defaultPolicy: 'channel_users',
channels: {},
...(shellApprovalDisabled ? { tools: { shell_exec: false } } : {}),
},
config: {
maxTaskDuration: 30_000,
maxToolCallsPerTask: 10,
autoStart: true,
taskDelay: 0,
approvalTimeout: 30_000,
},
coreEvents: new SimpleEventEmitter(),
cronScheduler: null,
taskLifecycle: lifecycle,
onApprovalNeeded,
onComplete: vi.fn(),
onFail,
dynamicEnvProvider: () => ({ ...dynamicEnv }),
}
return {
executor: new TaskExecutor(task, deps),
providerCalls,
advertisedTools,
onFail,
onApprovalNeeded,
callerWorkspace,
root,
store,
storeInitError,
}
}
it.each([
['cron source', { source: 'cron' as TaskSource }],
['internal source', { source: 'internal' as TaskSource, approvalEnabled: false }],
['approval disabled', { source: 'channel' as TaskSource, approvalEnabled: false }],
['shell approval disabled', { source: 'channel' as TaskSource, shellApprovalDisabled: true }],
] as const)(
'runs managed shell on the caller root while the GFS store needs recovery: %s',
async (_name, options) => {
const scenario = await shellScenario(options)
expect(scenario.store.isAvailable()).toBe(false)
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
expect(JSON.stringify(scenario.providerCalls)).toContain(scenario.callerWorkspace)
expect(spawn).toHaveBeenCalledOnce()
}
)
it('X1: a store left by the pre-#1028 image no longer blocks an approved shell or GFS delivery', async () => {
const scenario = await shellScenario({
source: 'channel',
preRedesign: true,
channelCaller: true,
})
expect(scenario.store.isAvailable()).toBe(true)
await scenario.executor.run()
expect(scenario.executor.executorState).toBe('waiting_approval')
expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)
expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')
expect(scenario.advertisedTools[0]).toContain('shell_exec')
await scenario.executor.resumeAfterApproval(false)
expect(scenario.onFail).not.toHaveBeenCalled()
expect(spawn).toHaveBeenCalledOnce()
const toolResult = JSON.stringify(scenario.providerCalls.at(-1))
expect(toolResult).toContain(scenario.callerWorkspace)
expect(toolResult).not.toContain('Processing lease failed')
expect(toolResult).not.toContain('download_busy')
})
it('X2: a store at workspace_unavailable keeps shell and GFS reads but withdraws delivery', async () => {
const scenario = await shellScenario({
source: 'cron',
calls: [
{
id: 'gfs-read-large',
name: 'clerum__gfs_read',
arguments: { drive: 'main', resourceId: largeResourceId },
},
{ id: 'shell-safety', name: 'shell_exec', arguments: { command: 'pwd' } },
],
})
// Witness: the store refused to initialize with exactly workspace_unavailable,
// not with any other initialize failure.
expect(scenario.storeInitError).toBeInstanceOf(GfsDownloadStoreError)
expect(scenario.storeInitError).toMatchObject({ code: 'workspace_unavailable' })
expect(scenario.store.isAvailable()).toBe(false)
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
const firstTools = scenario.advertisedTools[0]!
expect(firstTools).toEqual(
expect.arrayContaining([
'shell_exec',
'clerum__gfs_read',
'clerum__gfs_list',
'clerum__gfs_stat',
])
)
expect(firstTools).not.toContain('clerum__gfs_download')
expect(gfsClient.readMetadata).toHaveBeenCalledOnce()
expect(gfsClient.download).not.toHaveBeenCalled()
expect(JSON.stringify(scenario.providerCalls[1])).toContain('workspace_delivery_unavailable')
expect(spawn).toHaveBeenCalledOnce()
expect(JSON.stringify(scenario.providerCalls.at(-1))).toContain(scenario.callerWorkspace)
})
it('X2b: a store at workspace_unavailable still answers admitted inline reads, accessible and resolve', async () => {
const smallResourceId = 'd'.repeat(32)
const smallSource = {
kind: 'gfs' as const,
drive: 'main',
resourceId: smallResourceId,
gfsUri: `gfs://main/${smallResourceId}`,
name: 'small.txt',
version: 3,
}
const smallText = 'inline-admitted-sentinel region=north total=42'
const accessibleUri = 'gfs://main/accessible-sentinel'
const resolvedPath = '/reports/resolve-sentinel'
gfsClient.readMetadata.mockImplementation(async () => ({
source: structuredClone(smallSource),
size: Buffer.byteLength(smallText),
}))
const releaseReservation = vi.fn()
gfsClient.read.mockImplementation(async () => ({
source: structuredClone(smallSource),
bytes: Buffer.from(smallText, 'utf8'),
reservation: { release: releaseReservation },
}))
gfsClient.accessible.mockImplementation(async () => ({
items: [{ gfsUri: accessibleUri, permissions: ['read'] }],
}))
gfsClient.resolve.mockImplementation(async () => ({
gfsUri: accessibleUri,
pathCache: resolvedPath,
}))
const scenario = await shellScenario({
source: 'cron',
calls: [
{
id: 'gfs-read-small',
name: 'clerum__gfs_read',
arguments: { drive: 'main', resourceId: smallResourceId },
},
{ id: 'gfs-accessible', name: 'clerum__gfs_accessible', arguments: { drive: 'main' } },
{ id: 'gfs-resolve', name: 'clerum__gfs_resolve', arguments: { uri: accessibleUri } },
],
})
// Same fixture as X2: the store refused to initialize with workspace_unavailable.
expect(scenario.storeInitError).toBeInstanceOf(GfsDownloadStoreError)
expect(scenario.storeInitError).toMatchObject({ code: 'workspace_unavailable' })
expect(scenario.store.isAvailable()).toBe(false)
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
const firstTools = scenario.advertisedTools[0]!
expect(firstTools).toEqual(
expect.arrayContaining(['clerum__gfs_read', 'clerum__gfs_accessible', 'clerum__gfs_resolve'])
)
expect(firstTools).not.toContain('clerum__gfs_download')
// Four model turns: three tool calls and the closing answer.
expect(scenario.providerCalls).toHaveLength(4)
const toolResult = (turn: number, toolCallId: string) => {
const message = scenario.providerCalls[turn]!.find(
m => m.role === 'tool' && m.tool_call_id === toolCallId
)
if (!message) throw new Error(`No tool result for ${toolCallId} in turn ${turn}`)
return typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
}
// (a) The inline read went through metadata admission, then the in-memory
// read, and its exact text reached the model; delivery was never attempted.
expect(gfsClient.readMetadata).toHaveBeenCalledOnce()
expect(gfsClient.readMetadata.mock.calls[0]![0]).toMatchObject({ resourceId: smallResourceId })
expect(gfsClient.read).toHaveBeenCalledOnce()
expect(gfsClient.read.mock.calls[0]![0]).toMatchObject({ resourceId: smallResourceId })
expect(releaseReservation).toHaveBeenCalledOnce()
expect(gfsClient.download).not.toHaveBeenCalled()
const readResult = toolResult(1, 'gfs-read-small')
expect(readResult).toContain(smallText)
expect(readResult).not.toContain('workspace_delivery_unavailable')
// (b) accessible and resolve return their normal client result.
expect(gfsClient.accessible).toHaveBeenCalledOnce()
expect(gfsClient.accessible.mock.calls[0]![0]).toMatchObject({ drive: 'main' })
expect(toolResult(2, 'gfs-accessible')).toContain(accessibleUri)
expect(gfsClient.resolve).toHaveBeenCalledOnce()
expect(gfsClient.resolve.mock.calls[0]![0]).toMatchObject({ uri: accessibleUri })
expect(toolResult(3, 'gfs-resolve')).toContain(resolvedPath)
})
it.each([
['cron source', { source: 'cron' as TaskSource }],
['internal source', { source: 'internal' as TaskSource, approvalEnabled: false }],
['approval disabled', { source: 'channel' as TaskSource, approvalEnabled: false }],
['shell consent disabled', { source: 'channel' as TaskSource, shellApprovalDisabled: true }],
] as const)(
'keeps healthy non-GFS shell behavior on the trusted system root: %s',
async (_name, options) => {
const scenario = await shellScenario({ ...options, healthy: true })
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
expect(JSON.stringify(scenario.providerCalls)).toContain(scenario.callerWorkspace)
expect(spawn).toHaveBeenCalledOnce()
}
)
it.each(['cron', 'internal'] as const)(
'denies %s shell when the trusted caller root is absent',
async source => {
const scenario = await shellScenario({
source,
healthy: true,
approvalEnabled: false,
missingCallerRoot: true,
})
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
expect(JSON.stringify(scenario.providerCalls)).toContain('no verified caller workspace')
expect(spawn).not.toHaveBeenCalled()
}
)
// ---------------------------------------------------------------------------
// Vacuity falsifiers (#1019 / #1021). Before #1019, TaskExecutor built a
// processing-lease provider from the store and handed it to the shell, so a
// registry-level test cannot reproduce the pre-#1019 behaviour on its own.
// These four tests drive TaskExecutor.run() and must go red when every #1019
// production file is reverted.
// ---------------------------------------------------------------------------
/**
* Spy on every GfsDownloadStore method, enumerated at runtime so a method that
* exists only in another revision (the pre-#1019 processing-lease methods, for
* example) is still covered.
*/
function storeSpyNet(store: GfsDownloadStore) {
const names = Object.getOwnPropertyNames(GfsDownloadStore.prototype).filter(
name =>
name !== 'constructor' &&
typeof Object.getOwnPropertyDescriptor(GfsDownloadStore.prototype, name)?.value === 'function'
)
const spies = new Map(
names.map(name => {
const spy = vi.spyOn(
store as unknown as Record<string, (...args: unknown[]) => unknown>,
name
)
installedSpies.push(spy)
return [name, spy] as const
})
)
const counts = () => new Map([...spies].map(([name, spy]) => [name, spy.mock.calls.length]))
return { names, spies, counts }
}
/**
* Record, for every ShellTool.execute call, the timeout the executor passed and
* the store methods called between entering and leaving execute.
*/
function shellExecuteWindows(net: ReturnType<typeof storeSpyNet>, timeline?: string[]) {
const windows: Array<{ timeoutMs: number | undefined; storeCalls: string[] }> = []
const original = ShellTool.prototype.execute
const spy = vi.spyOn(ShellTool.prototype, 'execute').mockImplementation(async function (
this: ShellTool,
params,
context
) {
timeline?.push('shell execute entered')
const before = net.counts()
try {
return await original.call(this, params, context)
} finally {
timeline?.push('shell execute returned')
const after = net.counts()
windows.push({
timeoutMs: context?.timeoutMs,
storeCalls: [...after]
.filter(([name, calls]) => calls > (before.get(name) ?? 0))
.map(([name]) => name),
})
}
})
installedSpies.push(spy)
return windows
}
/**
* Record the store methods called during the whole TaskExecutor dispatch of
* every shell_exec call: the window opens when the scripted model returns the
* call to the executor and closes when the executor sends the next model turn,
* which carries the tool result. Everything the executor does for that call
* (tool loop, approval gate, registry lookup, ShellTool.execute, output
* processing) falls inside it. Registry construction before the first turn and
* the terminal retention settlement after the last turn fall outside it by
* construction, because no shell call is in flight then.
*/
function shellDispatchWindows(timeline: string[]) {
let net: ReturnType<typeof storeSpyNet> | undefined
let open: { toolCallId: string; before: Map<string, number> } | undefined
const windows: Array<{ toolCallId: string; storeCalls: string[] }> = []
const liveNet = () => {
if (!net) throw new Error('shellDispatchWindows: the store spy net was never attached')
return net
}
const observer: TurnObserver = {
turnStarted(turn) {
if (open) {
const after = liveNet().counts()
const before = open.before
windows.push({
toolCallId: open.toolCallId,
storeCalls: [...after]
.filter(([name, calls]) => calls > (before.get(name) ?? 0))
.map(([name]) => name),
})
open = undefined
}
timeline.push(`turn ${turn} started`)
},
turnReturned(turn, toolCall) {
timeline.push(`turn ${turn} returned ${toolCall?.name ?? 'final answer'}`)
if (toolCall?.name === 'shell_exec')
open = { toolCallId: toolCall.id, before: liveNet().counts() }
},
}
return {
observer,
windows,
attach(spyNet: ReturnType<typeof storeSpyNet>) {
net = spyNet
},
/** A window still open after run() means the tool result never reached the model. */
unclosed: () => open?.toolCallId,
}
}
it('X1-shell: a store left by the pre-#1028 image does not stop an approved shell (#1019)', async () => {
const scenario = await shellScenario({
source: 'channel',
preRedesign: true,
channelCaller: true,
})
await scenario.executor.run()
// Witness: the executor reached the live approval gate for shell_exec.
expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)
await scenario.executor.resumeAfterApproval(false)
// Claim: the approved command spawned in the caller root and its stdout returned.
expect(spawn).toHaveBeenCalledOnce()
expect(vi.mocked(spawn).mock.calls[0]![2]).toMatchObject({ cwd: scenario.callerWorkspace })
expect(JSON.stringify(scenario.providerCalls.at(-1))).toContain(scenario.callerWorkspace)
expect(scenario.onFail).not.toHaveBeenCalled()
})
it('X1-delivery: a store left by the pre-#1028 image is ignored and GFS delivery stays advertised (#1019)', async () => {
const scenario = await shellScenario({
source: 'channel',
preRedesign: true,
channelCaller: true,
})
await scenario.executor.run()
// Witness: the provider received a tool list built by the executor's registry.
expect(scenario.advertisedTools.length).toBeGreaterThan(0)
expect(scenario.advertisedTools[0]).toContain('shell_exec')
// Claim: delivery is advertised because the store came up available.
expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')
expect(scenario.store.isAvailable()).toBe(true)
})
/**
* A shell call whose stdout marker never appears verbatim in the command text,
* so finding the marker in the tool result proves the command ran.
*/
function markerCall(id: string, prefix: string): { call: ToolCall; marker: string } {
return {
call: { id, name: 'shell_exec', arguments: { command: `printf '%s_%s' ${prefix} STDOUT` } },
marker: `${prefix}_STDOUT`,
}
}
function shellToolResult(providerCalls: ChatMessage[][], toolCallId: string): string {
const message = providerCalls.flat().find(m => m.role === 'tool' && m.tool_call_id === toolCallId)
if (!message) throw new Error(`No tool result for ${toolCallId}`)
return typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
}
async function livePreRedesignScenario(call: ToolCall) {
let seededLease: { expiresAt: string } | undefined
const scenario = await shellScenario({
source: 'channel',
channelCaller: true,
calls: [call],
storeFixture: async () => {
const fixture = await preRedesignStore({ live: true })
seededLease = fixture.seededLease
return fixture
},
})
// Witness: the seeded lease is live by its own expiry, so ignoring it cannot
// be explained by expiry.
expect(seededLease).toBeDefined()
expect(Date.parse(seededLease!.expiresAt)).toBeGreaterThan(Date.now())
return scenario
}
it('X1-shell-live: a store with a non-expired legacy lease left by the pre-#1028 image does not stop an approved shell (#1019)', async () => {
const { call, marker } = markerCall('shell-live', 'X1_SHELL_LIVE')
const scenario = await livePreRedesignScenario(call)
await scenario.executor.run()
// Witness: the executor reached the live approval gate for shell_exec.
expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)
expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')
await scenario.executor.resumeAfterApproval(false)
expect(spawn).toHaveBeenCalledOnce()
expect(vi.mocked(spawn).mock.calls[0]![2]).toMatchObject({ cwd: scenario.callerWorkspace })
expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)
expect(scenario.onFail).not.toHaveBeenCalled()
})
it('X1-delivery-live: a store with a non-expired legacy lease left by the pre-#1028 image is ignored and GFS delivery stays advertised (#1019)', async () => {
const { call, marker } = markerCall('shell-live-delivery', 'X1_DELIVERY_LIVE')
const scenario = await livePreRedesignScenario(call)
await scenario.executor.run()
// Witness: the provider received a tool list built by the executor's registry.
expect(scenario.advertisedTools.length).toBeGreaterThan(0)
expect(scenario.advertisedTools[0]).toContain('shell_exec')
expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')
expect(scenario.store.isAvailable()).toBe(true)
expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)
await scenario.executor.resumeAfterApproval(false)
expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)
expect(scenario.onFail).not.toHaveBeenCalled()
})
describe('U5-TE: TaskExecutor runs a managed shell with a timeout above the former lease ceiling (#1021)', () => {
it.each([
['exactly one hour', 3_600_000],
['one millisecond over one hour', 3_600_001],
])('%s', async (_label, timeoutMs) => {
Object.assign(appConfig.nativeTool, { shellTimeout: timeoutMs, toolTimeout: timeoutMs })
const scenario = await shellScenario({ source: 'cron', healthy: true })
const windows = shellExecuteWindows(storeSpyNet(scenario.store))
await scenario.executor.run()
// Witness: the effective timeout reached ShellTool.execute unchanged.
expect(windows).toHaveLength(1)
expect(windows[0]!.timeoutMs).toBe(timeoutMs)
// Claim: the command spawned and its stdout reached the model.
expect(spawn).toHaveBeenCalledOnce()
expect(JSON.stringify(scenario.providerCalls.at(-1))).toContain(scenario.callerWorkspace)
expect(scenario.onFail).not.toHaveBeenCalled()
})
})
it('X3-TE: a shell dispatch inside TaskExecutor makes zero GFS download store calls (#1019)', async () => {
const timeline: string[] = []
const dispatch = shellDispatchWindows(timeline)
const scenario = await shellScenario({
source: 'cron',
healthy: true,
turnObserver: dispatch.observer,
})
const net = storeSpyNet(scenario.store)
dispatch.attach(net)
const executeWindows = shellExecuteWindows(net, timeline)
await scenario.executor.run()
// Witnesses: the net is attached to the store instance the executor wired
// (isAvailable is read while the registry is built, outside the dispatch
// window), exactly one shell dispatch window opened and closed with
// ShellTool.execute nested inside it, and the command's stdout reached the model.
expect(net.names).toContain('isAvailable')
expect(net.spies.get('isAvailable')!.mock.calls.length).toBeGreaterThan(0)
expect(dispatch.unclosed()).toBeUndefined()
expect(dispatch.windows).toEqual([{ toolCallId: 'shell-safety', storeCalls: expect.any(Array) }])
expect(executeWindows).toHaveLength(1)
expect(timeline).toEqual([
'turn 0 started',
'turn 0 returned shell_exec',
'shell execute entered',
'shell execute returned',
'turn 1 started',
'turn 1 returned final answer',
])
expect(spawn).toHaveBeenCalledOnce()
expect(JSON.stringify(scenario.providerCalls.at(-1))).toContain(scenario.callerWorkspace)
// Claim: no store method ran from the moment the executor received the shell
// call until it returned the shell result to the model.
expect(dispatch.windows[0]!.storeCalls).toEqual([])
expect(executeWindows[0]!.storeCalls).toEqual([])
})
it('X3-TE-channel: an approved channel shell dispatch with GFS delivery on makes zero GFS download store calls (#1019)', async () => {
const timeline: string[] = []
const dispatch = shellDispatchWindows(timeline)
const { call, marker } = markerCall('shell-safety-channel', 'X3_CHANNEL')
const scenario = await shellScenario({
source: 'channel',
channelCaller: true,
healthy: true,
calls: [call],
turnObserver: dispatch.observer,
})
const net = storeSpyNet(scenario.store)
dispatch.attach(net)
const executeWindows = shellExecuteWindows(net, timeline)
await scenario.executor.run()
// Witnesses: the shell reached the live approval gate, and delivery is on:
// every store read the executor made while building the registry reported
// the store live, and clerum__gfs_download was advertised. The test reads the
// spy's results rather than calling the store, so the open window stays clean.
expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)
expect(spawn).not.toHaveBeenCalled()
const availability = net.spies.get('isAvailable')!.mock.results
expect(availability.length).toBeGreaterThan(0)
expect(availability.every(result => result.value === true)).toBe(true)
expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')
await scenario.executor.resumeAfterApproval(false)
// Witnesses: one dispatch window opened at the model's shell call and closed
// at the next turn, with ShellTool.execute nested inside it, and the
// command's stdout reached the model.
expect(dispatch.unclosed()).toBeUndefined()
expect(dispatch.windows).toEqual([{ toolCallId: call.id, storeCalls: expect.any(Array) }])
expect(executeWindows).toHaveLength(1)
expect(timeline).toEqual([
'turn 0 started',
'turn 0 returned shell_exec',
'shell execute entered',
'shell execute returned',
'turn 1 started',
'turn 1 returned final answer',
])
expect(spawn).toHaveBeenCalledOnce()
expect(vi.mocked(spawn).mock.calls[0]![2]).toMatchObject({ cwd: scenario.callerWorkspace })
expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)
expect(scenario.onFail).not.toHaveBeenCalled()
// Claim: no store method ran from the moment the executor received the shell
// call, through the approval and its resume, until the shell result returned.
expect(dispatch.windows[0]!.storeCalls).toEqual([])
expect(executeWindows[0]!.storeCalls).toEqual([])
})
/** A store that was constructed but never initialized. */
async function uninitializedStore(): Promise<StoreFixture> {
const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-uninitialized-'))
roots.push(root)
const store = new GfsDownloadStore(root)
stores.push(store)
// Witness: a store that never ran initialize() reports itself unavailable.
expect(store.isAvailable()).toBe(false)
return { root, store }
}
/** A store that initialized and was then closed. */
async function closedStore(): Promise<StoreFixture> {
const { root, store } = await healthyStore()
// Witness: the store was available before close, so close made it unavailable.
expect(store.isAvailable()).toBe(true)
await store.close()
return { root, store }
}
describe('X6: an approved shell runs while the store is unavailable', () => {
it.each([
['never initialized', uninitializedStore],
['store closed', closedStore],
['initialize rejected with workspace_unavailable', unavailableStore],
] as const)('%s', async (_label, storeFixture) => {
const { call, marker } = markerCall('shell-unavailable', 'X6_SHELL')
const timeline: string[] = []
const dispatch = shellDispatchWindows(timeline)
const scenario = await shellScenario({
source: 'cron',
calls: [call],
turnObserver: dispatch.observer,
storeFixture,
})
// Witness: the store the executor is wired to is unavailable.
expect(scenario.store.isAvailable()).toBe(false)
const net = storeSpyNet(scenario.store)
dispatch.attach(net)
const executeWindows = shellExecuteWindows(net, timeline)
await scenario.executor.run()
// Witnesses: the executor read the unavailable store while building the
// registry and withdrew delivery; exactly one shell dispatch window opened
// and closed with ShellTool.execute nested inside it.
expect(net.spies.get('isAvailable')!.mock.calls.length).toBeGreaterThan(0)
expect(scenario.advertisedTools[0]).toContain('shell_exec')
expect(scenario.advertisedTools[0]).not.toContain('clerum__gfs_download')
expect(dispatch.unclosed()).toBeUndefined()
expect(dispatch.windows).toEqual([{ toolCallId: call.id, storeCalls: expect.any(Array) }])
expect(executeWindows).toHaveLength(1)
expect(timeline).toEqual([
'turn 0 started',
'turn 0 returned shell_exec',
'shell execute entered',
'shell execute returned',
'turn 1 started',
'turn 1 returned final answer',
])
// Claim: the command ran and its stdout reached the model.
expect(spawn).toHaveBeenCalledOnce()
expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)
expect(scenario.onFail).not.toHaveBeenCalled()
// Claim: no store method ran while the shell call was dispatched.
expect(dispatch.windows[0]!.storeCalls).toEqual([])
expect(executeWindows[0]!.storeCalls).toEqual([])
})
})
it('X5-env: a NUL secret in the dynamic shell environment fails to start without exposing its value (#1020)', async () => {
const secretTail = 's3cr3t-marker-1028'
const secretFragments = ['s3cr3t', 'marker-1028', secretTail]
const envCall: ToolCall = {
id: 'shell-env-nul',
name: 'shell_exec',
arguments: { command: 'printf "%s" "$CLERUM_TEST_SECRET"' },
}
const logCalls: unknown[][] = []
for (const level of ['error', 'warn', 'info', 'debug'] as const) {
const spy = vi.spyOn(logger, level)
installedSpies.push(spy)
logCalls.push(spy.mock.calls as unknown as unknown[])
}
const loggedText = () =>
JSON.stringify(logCalls.flat(), (_key, value) =>
value instanceof Error ? { message: value.message, stack: value.stack } : value
)
const toolResult = (providerCalls: ChatMessage[][]) => {
const message = providerCalls[1]?.find(m => m.role === 'tool' && m.tool_call_id === envCall.id)
if (!message) throw new Error(`No tool result for ${envCall.id} in turn 1`)
return typeof message.content === 'string' ? message.content : JSON.stringify(message.content)
}
// Witness: the same dynamic variable without the NUL reaches the shell, so the
// provider is wired and the command would print the secret if it could start.
const clean = await shellScenario({
source: 'cron',
healthy: true,
calls: [envCall],
dynamicEnv: { CLERUM_TEST_SECRET: `a${secretTail}` },
})
await clean.executor.run()
expect(clean.onFail).not.toHaveBeenCalled()
expect(spawn).toHaveBeenCalledOnce()
expect(toolResult(clean.providerCalls)).toContain(`a${secretTail}`)
vi.mocked(spawn).mockClear()
for (const calls of logCalls) calls.splice(0)
const scenario = await shellScenario({
source: 'cron',
healthy: true,
calls: [envCall],
dynamicEnv: { CLERUM_TEST_SECRET: `a\0${secretTail}` },
})
await scenario.executor.run()
expect(scenario.onFail).not.toHaveBeenCalled()
// The model sees only the key, never the value, inside the tool-output envelope.
const result = toolResult(scenario.providerCalls)
expect(result).toBe(
[
'<tool_output name="shell_exec" sanitized="false">',
'Command failed to start: environment variable CLERUM_TEST_SECRET contains a NUL character',
'</tool_output>',
].join('\n')
)
expect(result).not.toContain('Tool execution failed:')
const modelVisible = JSON.stringify(scenario.providerCalls)
// Witness for the log assertions: the shell logged its own start failure.
expect(loggedText()).toContain('ENV_VALUE_CONTAINS_NUL')
for (const fragment of secretFragments) {
expect(result).not.toContain(fragment)
expect(modelVisible).not.toContain(fragment)
expect(loggedText()).not.toContain(fragment)
}
expect(loggedText()).not.toContain('Tool execution failed')
expect(spawn).not.toHaveBeenCalled()
})