Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(mcp-host): cover restart gaps for legacy processing leases
Alfredo Lopezfdd0f115ceea
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex d9d3505c7..5aa25a82b 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.518",+ "version": "0.1.519","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.518",+ "version": "0.1.519","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 79016774d..54395b65f 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.518",+ "version": "0.1.519","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/agent/__tests__/taskExecutor.gfsExecutionSafety.test.ts b/mcp-host/src/agent/__tests__/taskExecutor.gfsExecutionSafety.test.tsindex 56ce0c910..b47a13c16 100644--- a/mcp-host/src/agent/__tests__/taskExecutor.gfsExecutionSafety.test.ts+++ b/mcp-host/src/agent/__tests__/taskExecutor.gfsExecutionSafety.test.ts@@ -1,6 +1,7 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { spawn } from 'child_process'import * as fs from 'node:fs/promises'+import type { FileHandle } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import { config as appConfig } from '../../config'@@ -116,10 +117,11 @@ async function unavailableStore() {}/**- * A store whose ledger carries a processing lease written by a pre-#1019 Host- * that crashed: expired, empty, and owned by a foreign writer session.+ * A closed store whose ledger carries a processing lease written by a pre-#1019+ * Host that crashed: empty and owned by a foreign writer session. By default+ * the lease is expired; `live` seeds one whose expiry is still in the future.*/-async function legacyLeaseStore() {+async function seedLegacyLeaseLedger({ live = false }: { live?: boolean } = {}) {const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-legacy-'))roots.push(root)const first = new GfsDownloadStore(root)@@ -128,23 +130,29 @@ async function legacyLeaseStore() {const ledgerPath = join(root, '.gfs-download-store', 'ledger-v1.json')const ledger = JSON.parse(await fs.readFile(ledgerPath, 'utf8'))const leaseId = '77777777-7777-4777-8777-777777777777'- ledger.processingLeases = {- [leaseId]: {- leaseId,- callerIdentity: '_system',- recordIds: [],- acquiredAt: new Date(Date.now() - 120_000).toISOString(),- expiresAt: new Date(Date.now() - 60_000).toISOString(),- writerSessionId: '88888888-8888-4888-8888-888888888888',- },+ const seededLease = {+ leaseId,+ callerIdentity: '_system',+ recordIds: [],+ acquiredAt: new Date(Date.now() - 120_000).toISOString(),+ expiresAt: new Date(Date.now() + (live ? 600_000 : -60_000)).toISOString(),+ writerSessionId: '88888888-8888-4888-8888-888888888888',}+ ledger.processingLeases = { [leaseId]: seededLease }await fs.writeFile(ledgerPath, JSON.stringify(ledger), { mode: 0o600 })+ return { root, ledgerPath, seededLease }+}++async function legacyLeaseStore({ live = false }: { live?: boolean } = {}) {+ const { root, seededLease } = await seedLegacyLeaseLedger({ live })const store = new GfsDownloadStore(root)stores.push(store)await store.initialize()- return { root, store }+ return { root, store, seededLease }}+type StoreFixture = { root: string; store: GfsDownloadStore; initError?: unknown }+async function healthyStore() {const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-healthy-'))roots.push(root)@@ -171,6 +179,7 @@ async function shellScenario({calls = [{ id: 'shell-safety', name: 'shell_exec', arguments: { command: 'pwd' } }],turnObserver,dynamicEnv = {},+ storeFixture,}: {source: TaskSourceapprovalEnabled?: boolean@@ -183,15 +192,19 @@ async function shellScenario({turnObserver?: TurnObserver/** Operator-managed env the shell receives through `dynamicEnvProvider`. */dynamicEnv?: Record<string, string>+ /** Builds the store the executor is wired to; overrides the flags above. */+ storeFixture?: () => Promise<StoreFixture>}) {appConfig.enableApproval = approvalEnabled- const fixture = legacyLease- ? await legacyLeaseStore()- : healthy- ? await healthyStore()- : await unavailableStore()+ const fixture: StoreFixture = storeFixture+ ? await storeFixture()+ : legacyLease+ ? await legacyLeaseStore()+ : healthy+ ? await healthyStore()+ : await unavailableStore()const { root, store } = fixture- const storeInitError = 'initError' in fixture ? fixture.initError : undefined+ const storeInitError = fixture.initErrorconst sourceMessage = channelCaller? {sender: 'gfs-safety-caller',@@ -286,6 +299,7 @@ async function shellScenario({onFail,onApprovalNeeded,callerWorkspace,+ root,store,storeInitError,}@@ -637,6 +651,87 @@ it('X1-delivery: an inherited legacy lease is discarded and GFS delivery stays aexpect(scenario.store.isAvailable()).toBe(true)})+/**+ * A shell call whose stdout marker never appears verbatim in the command text,+ * so finding the marker in the tool result proves the command ran.+ */+function markerCall(id: string, prefix: string): { call: ToolCall; marker: string } {+ return {+ call: { id, name: 'shell_exec', arguments: { command: `printf '%s_%s' ${prefix} STDOUT` } },+ marker: `${prefix}_STDOUT`,+ }+}++function shellToolResult(providerCalls: ChatMessage[][], toolCallId: string): string {+ const message = providerCalls.flat().find(m => m.role === 'tool' && m.tool_call_id === toolCallId)+ if (!message) throw new Error(`No tool result for ${toolCallId}`)+ return typeof message.content === 'string' ? message.content : JSON.stringify(message.content)+}++async function liveLegacyLeaseScenario(call: ToolCall) {+ let seededLease: { expiresAt: string } | undefined+ const scenario = await shellScenario({+ source: 'channel',+ channelCaller: true,+ calls: [call],+ storeFixture: async () => {+ const fixture = await legacyLeaseStore({ live: true })+ seededLease = fixture.seededLease+ return fixture+ },+ })+ // Witness: the seeded lease is live by its own expiry, so its discard cannot+ // be explained by expiry.+ expect(seededLease).toBeDefined()+ expect(Date.parse(seededLease!.expiresAt)).toBeGreaterThan(Date.now())+ return scenario+}++async function persistedLedger(root: string): Promise<Record<string, unknown>> {+ return JSON.parse(+ await fs.readFile(join(root, '.gfs-download-store', 'ledger-v1.json'), 'utf8')+ ) as Record<string, unknown>+}++it('X1-shell-live: an inherited non-expired legacy lease does not stop an approved shell (#1019)', async () => {+ const { call, marker } = markerCall('shell-live', 'X1_SHELL_LIVE')+ const scenario = await liveLegacyLeaseScenario(call)+ await scenario.executor.run()+ // Witness: the executor reached the live approval gate for shell_exec.+ expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)+ expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')++ await scenario.executor.resumeAfterApproval(false)++ expect(spawn).toHaveBeenCalledOnce()+ expect(vi.mocked(spawn).mock.calls[0]![2]).toMatchObject({ cwd: scenario.callerWorkspace })+ expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)+ expect(scenario.onFail).not.toHaveBeenCalled()+ const onDisk = await persistedLedger(scenario.root)+ expect(onDisk).toHaveProperty('records')+ expect(onDisk).not.toHaveProperty('processingLeases')+})++it('X1-delivery-live: an inherited non-expired legacy lease is discarded and GFS delivery stays advertised (#1019)', async () => {+ const { call, marker } = markerCall('shell-live-delivery', 'X1_DELIVERY_LIVE')+ const scenario = await liveLegacyLeaseScenario(call)+ await scenario.executor.run()+ // Witness: the provider received a tool list built by the executor's registry.+ expect(scenario.advertisedTools.length).toBeGreaterThan(0)+ expect(scenario.advertisedTools[0]).toContain('shell_exec')+ expect(scenario.advertisedTools[0]).toContain('clerum__gfs_download')+ expect(scenario.store.isAvailable()).toBe(true)+ expect(scenario.onApprovalNeeded).toHaveBeenCalledTimes(1)++ await scenario.executor.resumeAfterApproval(false)++ expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)+ expect(scenario.onFail).not.toHaveBeenCalled()+ const onDisk = await persistedLedger(scenario.root)+ expect(onDisk).toHaveProperty('records')+ expect(onDisk).not.toHaveProperty('processingLeases')+})+describe('U5-TE: TaskExecutor runs a managed shell with a timeout above the former lease ceiling (#1021)', () => {it.each([['exactly one hour', 3_600_000],@@ -697,6 +792,133 @@ it('X3-TE: a shell dispatch inside TaskExecutor makes zero GFS download store caexpect(executeWindows[0]!.storeCalls).toEqual([])})+/** Another live writer holds the store directory, so this store cannot initialize. */+async function contendedStore(): Promise<StoreFixture> {+ const root = await fs.mkdtemp(join(tmpdir(), 'gfs-shell-safety-contended-'))+ roots.push(root)+ const holder = new GfsDownloadStore(root)+ stores.push(holder)+ await holder.initialize()+ const store = new GfsDownloadStore(root)+ stores.push(store)+ const initError = await store.initialize().then(+ () => {+ throw new Error('contendedStore fixture: initialize() resolved while another writer held it')+ },+ (error: unknown) => error+ )+ // Witness: the holder is the live writer the contended store lost to.+ expect(holder.isAvailable()).toBe(true)+ expect(initError).toMatchObject({ code: 'writer_locked', transientWriterContention: true })+ return { root, store, initError }+}++/** A store that initialized and was then closed. */+async function closedStore(): Promise<StoreFixture> {+ const { root, store } = await healthyStore()+ // Witness: the store was available before close, so close made it unavailable.+ expect(store.isAvailable()).toBe(true)+ await store.close()+ return { root, store }+}++/**+ * A store whose initialize rejects because the persist of the legacy-lease+ * discard fails with EIO before the ledger rename.+ */+async function discardPersistFailedStore(): Promise<StoreFixture> {+ const { root, ledgerPath } = await seedLegacyLeaseLedger()+ const probe = await fs.open(ledgerPath, 'r')+ const prototype = Object.getPrototypeOf(probe) as FileHandle+ await probe.close()+ const originalWriteFile = prototype.writeFile+ let failedLedgerWrite: Record<string, unknown> | undefined+ const writeSpy = vi.spyOn(prototype, 'writeFile').mockImplementation(async function (+ this: FileHandle,+ ...args: Parameters<FileHandle['writeFile']>+ ) {+ const [data] = args+ if (failedLedgerWrite === undefined && typeof data === 'string') {+ const written = JSON.parse(data) as Record<string, unknown>+ if ('records' in written && !('processingLeases' in written)) {+ failedLedgerWrite = written+ throw Object.assign(new Error('injected ledger write failure'), { code: 'EIO' })+ }+ }+ return originalWriteFile.apply(this, args)+ })+ const store = new GfsDownloadStore(root)+ stores.push(store)+ let initError: unknown+ try {+ initError = await store.initialize().then(+ () => {+ throw new Error('discardPersistFailedStore fixture: initialize() resolved')+ },+ (error: unknown) => error+ )+ } finally {+ writeSpy.mockRestore()+ }+ // Witness: the failing write was the discard's own lease-free ledger, and the+ // lease is still on disk.+ expect(initError).toMatchObject({ code: 'EIO', message: 'injected ledger write failure' })+ expect(failedLedgerWrite).toBeDefined()+ expect(failedLedgerWrite).not.toHaveProperty('processingLeases')+ expect(JSON.parse(await fs.readFile(ledgerPath, 'utf8'))).toHaveProperty('processingLeases')+ return { root, store, initError }+}++describe('X6: an approved shell runs while the store is unavailable', () => {+ it.each([+ ['writer contention', contendedStore],+ ['store closed', closedStore],+ ['initialize rejected by an EIO persist during the legacy discard', discardPersistFailedStore],+ ] as const)('%s', async (_label, storeFixture) => {+ const { call, marker } = markerCall('shell-unavailable', 'X6_SHELL')+ const timeline: string[] = []+ const dispatch = shellDispatchWindows(timeline)+ const scenario = await shellScenario({+ source: 'cron',+ calls: [call],+ turnObserver: dispatch.observer,+ storeFixture,+ })+ // Witness: the store the executor is wired to is unavailable.+ expect(scenario.store.isAvailable()).toBe(false)+ const net = storeSpyNet(scenario.store)+ dispatch.attach(net)+ const executeWindows = shellExecuteWindows(net, timeline)++ await scenario.executor.run()++ // Witnesses: the executor read the unavailable store while building the+ // registry and withdrew delivery; exactly one shell dispatch window opened+ // and closed with ShellTool.execute nested inside it.+ expect(net.spies.get('isAvailable')!.mock.calls.length).toBeGreaterThan(0)+ expect(scenario.advertisedTools[0]).toContain('shell_exec')+ expect(scenario.advertisedTools[0]).not.toContain('clerum__gfs_download')+ expect(dispatch.unclosed()).toBeUndefined()+ expect(dispatch.windows).toEqual([{ toolCallId: call.id, storeCalls: expect.any(Array) }])+ expect(executeWindows).toHaveLength(1)+ expect(timeline).toEqual([+ 'turn 0 started',+ 'turn 0 returned shell_exec',+ 'shell execute entered',+ 'shell execute returned',+ 'turn 1 started',+ 'turn 1 returned final answer',+ ])+ // Claim: the command ran and its stdout reached the model.+ expect(spawn).toHaveBeenCalledOnce()+ expect(shellToolResult(scenario.providerCalls, call.id)).toContain(marker)+ expect(scenario.onFail).not.toHaveBeenCalled()+ // Claim: no store method ran while the shell call was dispatched.+ expect(dispatch.windows[0]!.storeCalls).toEqual([])+ expect(executeWindows[0]!.storeCalls).toEqual([])+ })+})+it('X5-env: a NUL secret in the dynamic shell environment fails to start without exposing its value (#1020)', async () => {const secretTail = 's3cr3t-marker-1028'const secretFragments = ['s3cr3t', 'marker-1028', secretTail]diff --git a/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.tsindex 2ec44d12f..5561ad984 100644--- a/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts+++ b/mcp-host/src/internalTools/gfsDownloadStoreRecovery.test.ts@@ -251,7 +251,73 @@ describe('legacy processing leases after #1019', () => {expect(reopened.debugUsage()).toMatchObject({ bytes: 7, files: 1 })})- it('S2: re-verifies records a discarded lease protected, reusing the intact copy and quarantining the altered one', async () => {+ it('S1c: discards several legacy leases from different foreign sessions in one initialize and counts each', async () => {+ const store = freshStore()+ await store.initialize()+ const protectedReceipt = await fixture(store)+ await store.close()+ const leaseIds = [+ '55555555-5555-4555-8555-555555555551',+ '55555555-5555-4555-8555-555555555552',+ '55555555-5555-4555-8555-555555555553',+ ]+ const sessions = [+ '66666666-6666-4666-8666-666666666661',+ '66666666-6666-4666-8666-666666666662',+ '66666666-6666-4666-8666-666666666663',+ ]+ const raw = await seedLegacyLeases([+ // Expired, protecting the completed record.+ legacyLease({+ leaseId: leaseIds[0],+ writerSessionId: sessions[0],+ recordIds: [protectedReceipt.id],+ }),+ // Not expired, protecting the same record.+ legacyLease({+ leaseId: leaseIds[1],+ writerSessionId: sessions[1],+ recordIds: [protectedReceipt.id],+ expiresAt: new Date(Date.now() + 600_000).toISOString(),+ }),+ // Not expired, protecting no record.+ legacyLease({+ leaseId: leaseIds[2],+ writerSessionId: sessions[2],+ expiresAt: new Date(Date.now() + 600_000).toISOString(),+ }),+ ])+ // Witness: the ledger the store reads carries three leases from three+ // distinct foreign sessions, one expired and two still live.+ const seeded = Object.values(JSON.parse(raw).processingLeases) as Array<{+ writerSessionId: string+ expiresAt: string+ }>+ expect(seeded).toHaveLength(3)+ expect(new Set(seeded.map(lease => lease.writerSessionId)).size).toBe(3)+ expect(seeded.filter(lease => Date.parse(lease.expiresAt) > Date.now())).toHaveLength(2)+ const warn = vi.spyOn(logger, 'warn')+ const before = await counterValue(DISCARDED_COUNTER)+ const reopened = freshStore()+ await reopened.initialize()+ const onDisk = JSON.parse(await fs.readFile(ledgerFile(), 'utf8'))+ expect(onDisk).not.toHaveProperty('processingLeases')+ expect(Object.keys(onDisk.records)).toEqual([protectedReceipt.id])+ expect((await counterValue(DISCARDED_COUNTER)) - before).toBe(3)+ expect(discardWarnings(warn.mock.calls)).toEqual([+ [+ { component: 'GfsDownloadStore', discarded: 3 },+ 'GFS download store discarded 3 legacy processing lease(s) at initialize',+ ],+ ])+ expect(reopened.isAvailable()).toBe(true)+ expect(reopened.debugRecord(protectedReceipt.id)?.state).toBe('completed')+ const receipt = await fixture(reopened, undefined, otherSource)+ expect(reopened.debugRecord(receipt.id)?.state).toBe('completed')+ expect(reopened.debugUsage()).toMatchObject({ bytes: 14, files: 2 })+ })++ it('S2:re-verifies records a discarded lease protected, reusing the intact copy and quarantining the altered one', async () => {const store = freshStore()await store.initialize()const intact = await fixture(store)@@ -495,6 +561,56 @@ describe('legacy processing leases after #1019', () => {expect(quarantineWarnings()).toHaveLength(2)expect(next.debugUsage().bytes).toBe(7)})++ it('S10: a legacy lease next to a half-finished transfer: the lease is discarded and the transfer is quarantined in the same initialize', async () => {+ const { child, message } = await childStore('transfer')+ expect(message.transferId).toMatch(/^[0-9a-f-]{36}$/)+ await kill(child)+ const raw = await seedLegacyLeases([legacyLease({ recordIds: [message.transferId] })])+ // Witness: the ledger the store reads carries both the lease and the+ // interrupted transfer, whose host partial is still on disk.+ const seeded = JSON.parse(raw)+ expect(seeded.processingLeases[LEGACY_LEASE_ID].recordIds).toEqual([message.transferId])+ const seededRecord = seeded.records[message.transferId]+ expect(seededRecord.state).toBe('transferring')+ const hostPartial = path.join(root, `${seededRecord.hostPath}.partial`)+ expect((await fs.lstat(hostPartial)).isFile()).toBe(true)++ const warn = vi.spyOn(logger, 'warn')+ const before = await counterValue(DISCARDED_COUNTER)+ const recovering = freshStore()+ await recovering.initialize()+ expect(recovering.isAvailable()).toBe(true)+ expect(recovering.debugRecord(message.transferId)?.state).toBe('quarantined')+ expect(recovering.debugUsage().bytes).toBe(7)+ expect((await counterValue(DISCARDED_COUNTER)) - before).toBe(1)+ expect(await counterValue(QUARANTINED_GAUGE)).toBe(1)+ expect(discardWarnings(warn.mock.calls)).toEqual([+ [+ { component: 'GfsDownloadStore', discarded: 1 },+ 'GFS download store discarded 1 legacy processing lease(s) at initialize',+ ],+ ])+ // The persisted ledger reflects both outcomes of the one initialize.+ const persisted = await fs.readFile(ledgerFile(), 'utf8')+ const onDisk = JSON.parse(persisted)+ expect(onDisk).not.toHaveProperty('processingLeases')+ expect(Object.keys(onDisk.records)).toEqual([message.transferId])+ expect(onDisk.records[message.transferId].state).toBe('quarantined')+ await recovering.close()++ // A second initialize finds nothing left to discard and rewrites nothing.+ const secondBefore = await counterValue(DISCARDED_COUNTER)+ const next = freshStore()+ await next.initialize()+ expect(next.isAvailable()).toBe(true)+ expect(next.debugRecord(message.transferId)?.state).toBe('quarantined')+ expect(next.debugUsage().bytes).toBe(7)+ expect((await counterValue(DISCARDED_COUNTER)) - secondBefore).toBe(0)+ expect(discardWarnings(warn.mock.calls)).toHaveLength(1)+ await next.close()+ expect(await fs.readFile(ledgerFile(), 'utf8')).toBe(persisted)+ })})describe('GFS writer recovery and filesystem invariants', () => {diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonindex 56f72873c..c0ad9904a 100644--- a/scripts/ci/pinned-vitest-cases/mcp-host.json+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -115,6 +115,34 @@"suite": "gfsDownloadStoreRecovery.test.ts","fullName": "legacy processing leases after #1019 S9: reports the current quarantined-record count after reconcile without double counting across boots"},+ {+ "suite": "gfsDownloadStoreRecovery.test.ts",+ "fullName": "legacy processing leases after #1019 S1c: discards several legacy leases from different foreign sessions in one initialize and counts each"+ },+ {+ "suite": "gfsDownloadStoreRecovery.test.ts",+ "fullName": "legacy processing leases after #1019 S10: a legacy lease next to a half-finished transfer: the lease is discarded and the transfer is quarantined in the same initialize"+ },+ {+ "suite": "taskExecutor.gfsExecutionSafety.test.ts",+ "fullName": "X1-shell-live: an inherited non-expired legacy lease does not stop an approved shell (#1019)"+ },+ {+ "suite": "taskExecutor.gfsExecutionSafety.test.ts",+ "fullName": "X1-delivery-live: an inherited non-expired legacy lease is discarded and GFS delivery stays advertised (#1019)"+ },+ {+ "suite": "taskExecutor.gfsExecutionSafety.test.ts",+ "fullName": "X6: an approved shell runs while the store is unavailable writer contention"+ },+ {+ "suite": "taskExecutor.gfsExecutionSafety.test.ts",+ "fullName": "X6: an approved shell runs while the store is unavailable store closed"+ },+ {+ "suite": "taskExecutor.gfsExecutionSafety.test.ts",+ "fullName": "X6: an approved shell runs while the store is unavailable initialize rejected by an EIO persist during the legacy discard"+ },{"suite": "shell.detachedStdio.test.ts","fullName": "ShellTool with stdio held by a process outside its group (#1028 P1) P1-a: a timeout with an escaped stdout/stderr holder settles through executeWithTimeout within the cleanup budget"