Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): open GFS store files without blocking on a FIFO
Alfredo Lopeze6bd277e3433
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex e6ef5233b..b8ef6d390 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.534",+ "version": "0.1.535","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.534",+ "version": "0.1.535","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 8d280d045..fdcf325cd 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.534",+ "version": "0.1.535","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/internalTools/gfsContentDownload.test.ts b/mcp-host/src/internalTools/gfsContentDownload.test.tsindex ee96c6847..c6a2e7e24 100644--- a/mcp-host/src/internalTools/gfsContentDownload.test.ts+++ b/mcp-host/src/internalTools/gfsContentDownload.test.ts@@ -1,8 +1,11 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { execFileSync } from 'node:child_process'import { createHash } from 'node:crypto'+import * as syncFs from 'node:fs'import * as fs from 'node:fs/promises'import { tmpdir } from 'node:os'import * as path from 'node:path'+import { setTimeout as delay } from 'node:timers/promises'import { register } from 'prom-client'import { downloadGfsContent } from './gfsContentDownload'import { readGfsMetadata } from './gfsContentRead'@@ -502,4 +505,39 @@ describe('governed GFS content download', () => {Buffer.from(bytes))})++ it('fails at once when source.partial is replaced by a FIFO before the download opens it', async () => {+ const createTransfer = store.createTransfer.bind(store)+ let fifo: string | undefined+ vi.spyOn(store, 'createTransfer').mockImplementationOnce(async input => {+ const transfer = await createTransfer(input)+ fifo = path.join(callerRoot, transfer.partialPath)+ await fs.rm(fifo)+ execFileSync('mkfifo', [fifo])+ return transfer+ })+ const bytes = Buffer.from('fifo partial')++ const download = downloadGfsContent(harness(bytes), args, options())+ const observed = download.then(+ () => 'downloaded',+ () => 'refused'+ )+ const outcome = await Promise.race([observed, delay(2_000).then(() => 'blocked')])+ if (outcome === 'blocked' && fifo !== undefined) {+ // Release the blocked open so the run ends; the assertion below fails.+ syncFs.closeSync(+ syncFs.openSync(fifo, syncFs.constants.O_RDONLY | syncFs.constants.O_NONBLOCK)+ )+ await observed+ }++ // Witness: the partial really was a FIFO when the download opened it.+ expect(fifo).toBeDefined()+ expect(outcome).toBe('refused')+ await expect(download).rejects.toMatchObject({ code: 'storage_write_failed' })+ // The next download of the same file works.+ const next = await downloadGfsContent(harness(bytes), args, options())+ await expect(store.readManagedFile(next.path, 'caller-a')).resolves.toEqual(bytes)+ })})diff --git a/mcp-host/src/internalTools/gfsContentDownload.ts b/mcp-host/src/internalTools/gfsContentDownload.tsindex a18013984..87a12dafa 100644--- a/mcp-host/src/internalTools/gfsContentDownload.ts+++ b/mcp-host/src/internalTools/gfsContentDownload.ts@@ -170,7 +170,12 @@ export async function downloadGfsContent(retentionOwnerId: options.retentionOwnerId,})const partialPath = path.join(options.callerWorkspacePath, transfer.partialPath)- destination = await fs.open(partialPath, constants.O_WRONLY | constants.O_NOFOLLOW)+ // O_NONBLOCK: a source.partial replaced by a FIFO fails the open (ENXIO+ // with no reader) instead of blocking the download.+ destination = await fs.open(+ partialPath,+ constants.O_WRONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK+ )const partialInfo = await destination.stat()if (!partialInfo.isFile() ||diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.tsindex 41d784be3..3e39be701 100644--- a/mcp-host/src/internalTools/gfsDownloadStore.ts+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts@@ -301,7 +301,11 @@ async function assertAbsent(target: string): Promise<void> {}async function syncDirectory(directory: string): Promise<void> {- const handle = await fs.open(directory, 'r')+ // A name swapped for a FIFO must fail, never block: O_DIRECTORY refuses it.+ const handle = await fs.open(+ directory,+ constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW | constants.O_NONBLOCK+ )try {await handle.sync()} finally {@@ -527,7 +531,11 @@ export class GfsDownloadStore {transfer.directoryIdentity = await fs.lstat(directory, { bigint: true })const partial = await fs.open(path.join(directory, PARTIAL_FILE),- constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,+ constants.O_WRONLY |+ constants.O_CREAT |+ constants.O_EXCL |+ constants.O_NOFOLLOW |+ constants.O_NONBLOCK,0o600)await partial.close()@@ -645,7 +653,11 @@ export class GfsDownloadStore {try {const handle = await fs.open(temporary,- constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,+ constants.O_WRONLY |+ constants.O_CREAT |+ constants.O_EXCL |+ constants.O_NOFOLLOW |+ constants.O_NONBLOCK,0o600)temporaryExists = true@@ -1189,9 +1201,13 @@ export class GfsDownloadStore {if (!info.isDirectory()) return { state: 'incomplete' }let raw: stringtry {+ // Anything in the tree may have been replaced from a shell. A FIFO with+ // no writer would block a plain open, and with it the sweep, startup and+ // every admission behind them; O_NONBLOCK opens it at once and the+ // descriptor's type decides.const handle = await fs.open(path.join(directory, META_FILE),- constants.O_RDONLY | constants.O_NOFOLLOW+ constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)try {const metaInfo = await handle.stat()diff --git a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.tsindex cf4fd20bb..d42fb093f 100644--- a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts+++ b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts@@ -6,10 +6,12 @@* a real temporary directory.*/import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { execFileSync } from 'node:child_process'import * as syncFs from 'node:fs'import * as fs from 'node:fs/promises'import { tmpdir } from 'node:os'import * as path from 'node:path'+import { setTimeout as delay } from 'node:timers/promises'import {callerDirectory,completedCopy,@@ -649,3 +651,47 @@ describe('GFS download store adversarial round: expiry, close, recency and removexpect(events.slice(events.indexOf('rename-source'))).toContain('open-directory')})})++describe('GFS download store adversarial round: a FIFO never blocks publication', () => {+ const KEY = 'dave-key'++ it('a download directory swapped for a FIFO before its sync fails the publication at once', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { transfer, bytes } = await startTransfer(store, root, KEY, 1, 12)+ const directory = syncFs.realpathSync(path.join(root, `.gfs-downloads/input-${transfer.id}`))+ let renamedSource = false+ let swapped = false+ injectFault('rename', target => {+ if (target.endsWith(`input-${transfer.id}/source.partial`)) renamedSource = true+ return undefined+ })+ injectFault('open', target => {+ if (!renamedSource || swapped || path.resolve(target) !== directory) return undefined+ swapped = true+ syncFs.renameSync(directory, `${directory}-moved`)+ execFileSync('mkfifo', [directory])+ return undefined+ })++ const publication = store.publish(transfer.id, KEY, digestOf(bytes))+ const observed = publication.then(+ () => 'published',+ () => 'refused'+ )+ const outcome = await Promise.race([observed, delay(2_000).then(() => 'blocked')])+ if (outcome === 'blocked') {+ // Release the blocked open so the run ends; the assertion below fails.+ syncFs.closeSync(+ syncFs.openSync(directory, syncFs.constants.O_WRONLY | syncFs.constants.O_NONBLOCK)+ )+ await observed+ }+ faults.clear()++ // Witness: the swap happened after the source rename, at the sync's open.+ expect(swapped).toBe(true)+ expect(syncFs.lstatSync(directory).isFIFO()).toBe(true)+ expect(outcome).toBe('refused')+ })+})diff --git a/mcp-host/src/internalTools/gfsDownloadStoreNonBlocking.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreNonBlocking.test.tsnew file mode 100644index 000000000..7fcc332a0--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreNonBlocking.test.ts@@ -0,0 +1,151 @@+/**+ * A FIFO left in the user-writable store tree must never block the GFS store.+ * Every case uses a real FIFO with no peer. If an open blocks anyway, the+ * bounded race reports `blocked`, the other end is opened once to release the+ * worker thread, and the assertion on the outcome fails the test.+ */+import { afterEach, beforeEach, describe, expect, it } from 'vitest'+import { execFileSync } from 'node:child_process'+import { randomUUID } from 'node:crypto'+import * as syncFs from 'node:fs'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import { setTimeout as delay } from 'node:timers/promises'+import {+ callerDirectory,+ completedCopy,+ downloadDirectory,+ exists,+} from '../__tests__/fixtures/gfsStoreTestKit'+import { bootstrapGfsRuntime } from '../gfsRuntime'+import { GfsDownloadStore } from './gfsDownloadStore'++const KEY = 'fifo-key'+const BOUND_MS = 2_000++let hostRoot: string+const stores: GfsDownloadStore[] = []++beforeEach(() => {+ hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-fifo-'))+})++afterEach(async () => {+ for (const opened of stores.splice(0)) await opened.close(0)+ await fs.rm(hostRoot, { recursive: true, force: true })+})++function mkfifo(target: string): void {+ execFileSync('mkfifo', [target])+ if (!syncFs.lstatSync(target).isFIFO()) throw new Error('mkfifo did not create a FIFO')+}++/** A download directory whose meta.json is a FIFO. */+function plantFifoMeta(root: string): string {+ const directory = downloadDirectory(root, randomUUID())+ syncFs.mkdirSync(directory, { recursive: true, mode: 0o700 })+ mkfifo(path.join(directory, 'meta.json'))+ return directory+}++/**+ * Races `pending` against BOUND_MS. A blocked open on `fifo` is released by+ * opening the other end without blocking, so the run never hangs.+ */+async function settleWithin(+ pending: Promise<unknown>,+ fifo: string,+ blockedEnd: 'reader' | 'writer'+): Promise<'settled' | 'blocked'> {+ const observed = pending.then(+ () => undefined,+ () => undefined+ )+ const outcome = await Promise.race([+ observed.then(() => 'settled' as const),+ delay(BOUND_MS).then(() => 'blocked' as const),+ ])+ if (outcome === 'blocked') {+ const peer = syncFs.openSync(+ fifo,+ (blockedEnd === 'reader' ? syncFs.constants.O_WRONLY : syncFs.constants.O_RDONLY) |+ syncFs.constants.O_NONBLOCK+ )+ syncFs.closeSync(peer)+ await observed+ }+ return outcome+}++async function openStore(): Promise<GfsDownloadStore> {+ const opened = new GfsDownloadStore(hostRoot)+ stores.push(opened)+ await opened.initialize()+ return opened+}++describe('GFS download store: a FIFO in the store tree never blocks', () => {+ it('Host bootstrap settles with a meta.json FIFO and removes its directory', async () => {+ const root = callerDirectory(hostRoot, KEY)+ const directory = plantFifoMeta(root)++ const pending = bootstrapGfsRuntime(hostRoot)+ const outcome = await settleWithin(pending, path.join(directory, 'meta.json'), 'reader')+ const runtime = await pending+ try {+ expect(outcome).toBe('settled')+ expect(runtime.store.isAvailable()).toBe(true)+ expect(exists(directory)).toBe(false)+ // Witness: an ordinary download, publication and managed read work.+ const copy = await completedCopy(runtime.store, root, KEY, 1, 32)+ await expect(runtime.store.readManagedFile(copy.receipt.path, KEY)).resolves.toEqual(+ copy.bytes+ )+ } finally {+ await runtime.stop()+ }+ })++ it('a runtime sweep settles with a meta.json FIFO and removes its directory', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const before = await completedCopy(store, root, KEY, 2, 24)+ const directory = plantFifoMeta(root)++ const sweep = store.cleanupExpired()+ const outcome = await settleWithin(sweep, path.join(directory, 'meta.json'), 'reader')++ expect(outcome).toBe('settled')+ await expect(sweep).resolves.toMatchObject({ removedIncomplete: 1 })+ expect(exists(directory)).toBe(false)+ // Witness: the published copy survives the sweep and a new admission works.+ await expect(store.readManagedFile(before.receipt.path, KEY)).resolves.toEqual(before.bytes)+ const after = await completedCopy(store, root, KEY, 3, 24)+ await expect(store.readManagedFile(after.receipt.path, KEY)).resolves.toEqual(after.bytes)+ })++ it('managed reads of a published copy whose source became a FIFO refuse without blocking', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const copy = await completedCopy(store, root, KEY, 4, 32)+ const source = path.join(root, copy.receipt.path)+ // Witness: the published copy is served before the swap.+ await expect(store.readManagedFile(copy.receipt.path, KEY)).resolves.toEqual(copy.bytes)+ syncFs.rmSync(source)+ mkfifo(source)++ const read = store.readManagedFile(copy.receipt.path, KEY)+ expect(await settleWithin(read, source, 'reader')).toBe('settled')+ await expect(read).rejects.toMatchObject({ code: 'download_missing' })+ const prefix = store.readManagedFilePrefix(copy.receipt.path, KEY, 8)+ expect(await settleWithin(prefix, source, 'reader')).toBe('settled')+ await expect(prefix).rejects.toMatchObject({ code: 'download_missing' })++ // The sweep treats it as incomplete and removes it; the store keeps working.+ await store.cleanupExpired()+ expect(exists(path.dirname(source))).toBe(false)+ const next = await completedCopy(store, root, KEY, 5, 32)+ await expect(store.readManagedFile(next.receipt.path, KEY)).resolves.toEqual(next.bytes)+ })+})diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonindex b04a06193..cb013c877 100644--- a/scripts/ci/pinned-vitest-cases/mcp-host.json+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -758,5 +758,25 @@{"suite": "gfsExecutionCapability.test.ts","fullName": "gfsStoreCallerIdentity gives a task without a source message the system key of its caller root"+ },+ {+ "suite": "gfsDownloadStoreNonBlocking.test.ts",+ "fullName": "GFS download store: a FIFO in the store tree never blocks Host bootstrap settles with a meta.json FIFO and removes its directory"+ },+ {+ "suite": "gfsDownloadStoreNonBlocking.test.ts",+ "fullName": "GFS download store: a FIFO in the store tree never blocks a runtime sweep settles with a meta.json FIFO and removes its directory"+ },+ {+ "suite": "gfsDownloadStoreNonBlocking.test.ts",+ "fullName": "GFS download store: a FIFO in the store tree never blocks managed reads of a published copy whose source became a FIFO refuse without blocking"+ },+ {+ "suite": "gfsDownloadStoreAdversarial.test.ts",+ "fullName": "GFS download store adversarial round: a FIFO never blocks publication a download directory swapped for a FIFO before its sync fails the publication at once"+ },+ {+ "suite": "gfsContentDownload.test.ts",+ "fullName": "governed GFS content download fails at once when source.partial is replaced by a FIFO before the download opens it"}]