Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): keep GFS store copies on transient I/O errors
Alfredo Lopez539c3d2c92bf
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 3d058f119..850fed973 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.529",+ "version": "0.1.530","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.529",+ "version": "0.1.530","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex a1d6c7858..d0ad8c6a6 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.529",+ "version": "0.1.530","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.tsindex ea0a60627..4e37bf417 100644--- a/mcp-host/src/internalTools/gfsDownloadStore.ts+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts@@ -11,7 +11,11 @@ import {GFS_HOST_ACTIVE_DOWNLOADS,GFS_HOST_RETAINED_FILES,} from './gfsFilePolicy'-import { openPrivateStoreObject, verifyPrivateStoreDirectory } from './gfsStorePrivateFiles'+import {+ PrivateStoreUntrustedError,+ openPrivateStoreObject,+ verifyPrivateStoreDirectory,+} from './gfsStorePrivateFiles'const GFS_DOWNLOAD_STORE_LOG_COMPONENT = 'GfsDownloadStore'@@ -157,9 +161,14 @@ export interface GfsDownloadInventory {byCaller: Map<string, { bytes: number; files: number }>}+/**+ * `unknown`: the disk did not answer (EMFILE, EIO, ...). Nothing is decided+ * about the directory; it is kept as it is and inspected again later.+ */type DirectoryState =| { state: 'absent' }| { state: 'incomplete' }+ | { state: 'unknown'; code: string }| { state: 'complete'; entry: Entry }function isValidSource(value: unknown): value is GfsImageSource {@@ -239,6 +248,38 @@ function errorCode(error: unknown): string {return typeof code === 'string' ? code : 'unknown'}+/**+ * Errnos that are an answer about the object itself rather than about the+ * process or the volume at this instant: it is gone, it is a symlink or sits+ * under a non-directory, or its mode refuses the owner.+ */+const DEFINITIVE_ERRNO = new Set(['ENOENT', 'ELOOP', 'ENOTDIR', 'EACCES'])++/**+ * True only when a failure proves that a copy or directory is not what the+ * store wrote: a size, inode or digest mismatch (raised as a store error), an+ * untrusted owner, mode or type, or a definitive errno. Anything else (EMFILE,+ * EIO, ENOMEM, ...) is transient: the object is kept and checked again later,+ * never removed on its account.+ */+function isDefinitiveMismatch(error: unknown): boolean {+ if (error instanceof GfsDownloadStoreError) return error.code !== 'publication_cancelled'+ if (error instanceof PrivateStoreUntrustedError) return true+ return DEFINITIVE_ERRNO.has(errorCode(error))+}++/** Errno, or the error class name for the store's own refusals; never a path. */+function diagnosticCode(error: unknown): string {+ const code = errorCode(error)+ return code === 'unknown' && error instanceof Error ? error.name : code+}++function stateFor(error: unknown): DirectoryState {+ return isDefinitiveMismatch(error)+ ? { state: 'incomplete' }+ : { state: 'unknown', code: errorCode(error) }+}+/** A removal is proven by ENOENT, not by `fs.rm` returning. */async function assertAbsent(target: string): Promise<void> {try {@@ -676,6 +717,7 @@ export class GfsDownloadStore {)if (!entry) return undefinedlet verified = false+ let transient: string | undefinedtry {const handle = await this.openEntryContent(entry)try {@@ -692,8 +734,18 @@ export class GfsDownloadStore {} catch (error) {if (error instanceof GfsDownloadStoreError && error.code === 'publication_cancelled')throw error+ if (!isDefinitiveMismatch(error)) transient = errorCode(error)verified = false}+ if (transient !== undefined) {+ // The disk did not answer: this is not evidence against the copy,+ // which another task may still hold. It stays; this caller downloads.+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: transient },+ 'GFS download store could not verify a published copy for reuse; it is kept and checked again later'+ )+ return undefined+ }if (!verified) {await this.removeEntry(entry, 'incomplete_removed')return undefined@@ -749,7 +801,8 @@ export class GfsDownloadStore {} finally {await handle.close()}- } catch {+ } catch (error) {+ this.reportManagedReadFailure(error)throw new GfsDownloadStoreError('download_missing')}if (createHash('sha256').update(bytes).digest('hex') !== entry.sha256) {@@ -781,7 +834,8 @@ export class GfsDownloadStore {} finally {await handle.close()}- } catch {+ } catch (error) {+ this.reportManagedReadFailure(error)throw new GfsDownloadStoreError('download_missing')}}@@ -927,6 +981,15 @@ export class GfsDownloadStore {this.forget(id)continue}+ if (state.state === 'unknown') {+ // Kept as it is, indexed or not; the next sweep inspects it again.+ recordGfsDownloadExpiry('sweep_failed')+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: state.code },+ 'GFS download store could not inspect a download directory; it is kept and the next sweep retries it'+ )+ continue+ }if (state.state === 'incomplete') {this.forget(id)await removeIncomplete(directory)@@ -1088,7 +1151,7 @@ export class GfsDownloadStore {info = await fs.lstat(directory)} catch (error) {if (errorCode(error) === 'ENOENT') return { state: 'absent' }- return { state: 'incomplete' }+ return stateFor(error)}if (!info.isDirectory()) return { state: 'incomplete' }let raw: string@@ -1104,16 +1167,16 @@ export class GfsDownloadStore {} finally {await handle.close()}- } catch {- return { state: 'incomplete' }+ } catch (error) {+ return stateFor(error)}const meta = parseMeta(raw, id, now)if (!meta) return { state: 'incomplete' }try {const source = await fs.lstat(path.join(directory, SOURCE_FILE))if (!source.isFile() || source.size !== meta.sizeBytes) return { state: 'incomplete' }- } catch {- return { state: 'incomplete' }+ } catch (error) {+ return stateFor(error)}return {state: 'complete',@@ -1127,19 +1190,23 @@ export class GfsDownloadStore {}}- /** An indexed entry is re-checked with one lstat; meta.json is not re-read. */+ /**+ * An indexed entry is re-checked with one lstat; meta.json is not re-read.+ * Only a definitive answer makes it incomplete; a transient errno keeps it.+ */private async recheckIndexed(entry: Entry): Promise<DirectoryState> {try {const source = await fs.lstat(path.join(entry.directory, SOURCE_FILE))if (source.isFile() && source.size === entry.sizeBytes) return { state: 'complete', entry }+ return { state: 'incomplete' }} catch (error) {- if (errorCode(error) === 'ENOENT') {- try {- await fs.lstat(entry.directory)- } catch (directoryError) {- if (errorCode(directoryError) === 'ENOENT') return { state: 'absent' }- }- }+ if (errorCode(error) !== 'ENOENT') return stateFor(error)+ }+ try {+ await fs.lstat(entry.directory)+ } catch (directoryError) {+ if (errorCode(directoryError) === 'ENOENT') return { state: 'absent' }+ return stateFor(directoryError)}return { state: 'incomplete' }}@@ -1206,9 +1273,13 @@ export class GfsDownloadStore {}/**- * A `.gfs-downloads` that is a symlink, not a directory, or not private to- * this process is removed without following it and recreated, so a shell- * command can cost a caller its cached copies but never block its downloads.+ * A `.gfs-downloads` proven not to be the store's (a symlink, not a+ * directory, gone, unreadable to its owner, or not owned by this process+ * with a private mode) is removed without following it and recreated, so a+ * shell command can cost a caller its cached copies but never block its+ * downloads. Any other failure to verify it (EMFILE, EIO, ...) proves+ * nothing: it is rethrown, the admission fails and the directory and its+ * copies stay.*/private async ensureDownloadsRoot(downloadsRoot: string): Promise<void> {try {@@ -1220,8 +1291,8 @@ export class GfsDownloadStore {try {await verifyPrivateStoreDirectory(downloadsRoot)return- } catch {- // Not a private directory: replaced below.+ } catch (error) {+ if (!isDefinitiveMismatch(error)) throw error}for (const id of this.idsUnder(downloadsRoot)) this.forget(id)await this.removeVerified(downloadsRoot)@@ -1406,6 +1477,20 @@ export class GfsDownloadStore {if (real !== parent) throw new GfsDownloadStoreError('workspace_unavailable')}+ /**+ * The caller always gets download_missing, so a read failure reveals+ * nothing about the copy. The owner's own published copy failing for any+ * reason but its absence is worth an operator's attention; code only.+ */+ private reportManagedReadFailure(error: unknown): void {+ if (errorCode(error) === 'ENOENT') return+ if (error instanceof GfsDownloadStoreError && error.code === 'download_missing') return+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: diagnosticCode(error) },+ 'GFS download store could not read a published copy for its caller'+ )+ }+private reportSweepFailure(error: unknown): void {recordGfsDownloadExpiry('sweep_failed')logger.warn(diff --git a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.tsindex be0635e7f..5eb91e4a1 100644--- a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts+++ b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts@@ -10,7 +10,17 @@ import * as syncFs from 'node:fs'import * as fs from 'node:fs/promises'import { tmpdir } from 'node:os'import * as path from 'node:path'-import { callerDirectory, completedCopy, sourceFor } from '../__tests__/fixtures/gfsStoreTestKit'+import {+ callerDirectory,+ completedCopy,+ exists,+ expiryCount,+ metaFor,+ plantDownload,+ sourceFor,+ startTransfer,+} from '../__tests__/fixtures/gfsStoreTestKit'+import { logger } from '../logger'import { deriveUserKey } from '../workspace/userKey'import { GfsDownloadStore } from './gfsDownloadStore'@@ -51,6 +61,36 @@ function injectFault(operation: FsOperation, fault: Fault): void {faults.set(operation, fault)}+/**+ * Watches `operation` on every path ending with `suffix`: records each call+ * and fails it with `code`, or only records it when `code` is undefined.+ */+function watch(operation: FsOperation, suffix: string, code?: string): string[] {+ const calls: string[] = []+ injectFault(operation, target => {+ if (!target.endsWith(suffix)) return undefined+ calls.push(target)+ return code+ })+ return calls+}++/** Every warning logged, as `{ code, message }`. */+function warnings(warn: {+ mock: { calls: unknown[][] }+}): Array<{ code: unknown; message: unknown }> {+ return warn.mock.calls.map(call => ({+ code: (call[0] as { code?: unknown }).code,+ message: call[1],+ }))+}++const REUSE_KEPT =+ 'GFS download store could not verify a published copy for reuse; it is kept and checked again later'+const SWEEP_KEPT =+ 'GFS download store could not inspect a download directory; it is kept and the next sweep retries it'+const READ_FAILED = 'GFS download store could not read a published copy for its caller'+let hostRoot: stringconst stores: GfsDownloadStore[] = []@@ -164,3 +204,130 @@ describe('GFS download store adversarial round: caller root shape (F9)', () => {).resolves.toMatchObject({ sizeBytes: 8 })})})++describe('GFS download store adversarial round: transient I/O is not corruption (F1/ADV-3)', () => {+ const KEY = 'alice-key'++ it('reuse that cannot open a pinned copy keeps it for the task that holds it', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { receipt, bytes } = await completedCopy(store, root, KEY, 1, 128, { owner: 'task-1' })+ const warn = vi.spyOn(logger, 'warn')+ const opens = watch('open', `input-${receipt.id}/source`, 'EMFILE')++ await expect(+ store.reusableReceipt(KEY, sourceFor(1), 128, { retentionOwnerId: 'task-2' })+ ).resolves.toBeUndefined()+ faults.clear()++ // Witness: reuse reached the copy and failed on it.+ expect(opens.length).toBeGreaterThan(0)+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: REUSE_KEPT })+ expect(exists(path.join(root, receipt.path))).toBe(true)+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)+ // Once the disk answers, the same copy is reused.+ await expect(store.reusableReceipt(KEY, sourceFor(1), 128)).resolves.toMatchObject({+ id: receipt.id,+ })+ })++ it('reuse still removes a copy whose bytes no longer match its receipt', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { receipt } = await completedCopy(store, root, KEY, 2, 64)+ const removed = await expiryCount('incomplete_removed')+ syncFs.writeFileSync(path.join(root, receipt.path), Buffer.alloc(64, 0xee))++ await expect(store.reusableReceipt(KEY, sourceFor(2), 64)).resolves.toBeUndefined()+ expect(exists(path.join(root, `.gfs-downloads/input-${receipt.id}`))).toBe(false)+ expect(await expiryCount('incomplete_removed')).toBe(removed + 1)+ })++ it('a sweep that cannot lstat a published copy keeps it and counts sweep_failed', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { receipt, bytes } = await completedCopy(store, root, KEY, 3, 96, { owner: 'task-1' })+ const swept = await expiryCount('sweep_failed')+ const warn = vi.spyOn(logger, 'warn')+ const lstats = watch('lstat', `input-${receipt.id}/source`, 'EMFILE')++ const result = await store.cleanupExpired()+ faults.clear()++ expect(lstats.length).toBeGreaterThan(0)+ expect(result).toMatchObject({ removedIncomplete: 0, removedExpired: 0, removeFailed: 0 })+ expect(await expiryCount('sweep_failed')).toBe(swept + 1)+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: SWEEP_KEPT })+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)+ // Contrast: a source that is really gone is removed by the next sweep.+ syncFs.rmSync(path.join(root, receipt.path))+ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })+ })++ it('a startup sweep that cannot open meta.json keeps the copy for the next start', async () => {+ const root = callerDirectory(hostRoot, KEY)+ const bytes = Buffer.alloc(100, 7)+ const id = '00000000-0000-4000-8000-000000000001'+ const planted = plantDownload(root, { id, bytes, meta: metaFor(id, bytes, Date.now()) })+ const opens = watch('open', '/meta.json', 'EMFILE')++ await openStore()+ faults.clear()++ expect(opens.length).toBeGreaterThan(0)+ expect(exists(planted.directory)).toBe(true)+ // Witness: a start that can read it adopts it.+ const second = await openStore()+ expect((await second.debugInventory()).byCaller.get(KEY)).toEqual({ files: 1, bytes: 100 })+ })++ it('an admission that cannot verify .gfs-downloads fails and keeps every copy', async () => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { receipt, bytes } = await completedCopy(store, root, KEY, 4, 80, { owner: 'task-1' })+ const opens = watch('open', `${KEY}/.gfs-downloads`, 'EMFILE')++ await expect(startTransfer(store, root, KEY, 5, 40)).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ faults.clear()++ expect(opens.length).toBeGreaterThan(0)+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)+ // Contrast: a `.gfs-downloads` with a non-private mode is replaced.+ syncFs.chmodSync(path.join(root, '.gfs-downloads'), 0o755)+ const replaced = await startTransfer(store, root, KEY, 6, 40)+ expect(exists(path.join(root, receipt.path))).toBe(false)+ expect(exists(path.join(root, replaced.transfer.partialPath))).toBe(true)+ })++ it.each([+ ['readManagedFile', (store: GfsDownloadStore, p: string) => store.readManagedFile(p, KEY)],+ [+ 'readManagedFilePrefix',+ (store: GfsDownloadStore, p: string) => store.readManagedFilePrefix(p, KEY),+ ],+ ])(+ '%s logs a non-ENOENT failure of the owner’s copy and still answers download_missing',+ async (_name, read) => {+ const store = await openStore()+ const root = callerDirectory(hostRoot, KEY)+ const { receipt } = await completedCopy(store, root, KEY, 7, 32)+ const warn = vi.spyOn(logger, 'warn')+ const failed = watch('open', `input-${receipt.id}/source`, 'EMFILE')++ await expect(read(store, receipt.path)).rejects.toMatchObject({ code: 'download_missing' })+ expect(failed).toHaveLength(1)+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: READ_FAILED })++ // ENOENT is the copy's absence: the same answer, nothing logged. The+ // witness is the open the read had to attempt.+ warn.mockClear()+ syncFs.rmSync(path.join(root, receipt.path))+ const attempted = watch('open', `input-${receipt.id}/source`)+ await expect(read(store, receipt.path)).rejects.toMatchObject({ code: 'download_missing' })+ expect(attempted).toHaveLength(1)+ expect(warnings(warn).map(entry => entry.message)).not.toContain(READ_FAILED)+ }+ )+})diff --git a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts b/mcp-host/src/internalTools/gfsStorePrivateFiles.tsindex c3e948c3a..a79a656ad 100644--- a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts+++ b/mcp-host/src/internalTools/gfsStorePrivateFiles.ts@@ -22,11 +22,23 @@ export function isPrivateStoreStatTrusted(info: Stats, kind: 'file' | 'directory)}+/**+ * The object was opened and proved not to be the store's: wrong owner, mode,+ * type or link count, or it changed while it was checked. Unlike an errno+ * such as EMFILE or EIO, this is a definitive answer about the object.+ */+export class PrivateStoreUntrustedError extends Error {+ constructor(message = 'Untrusted private-store inode') {+ super(message)+ this.name = 'PrivateStoreUntrustedError'+ }+}+/*** The opened inode is trusted, but the name now points at another inode. An* atomic rename over the name between open and lstat produces exactly this.*/-export class PrivateStoreNameMovedError extends Error {+export class PrivateStoreNameMovedError extends PrivateStoreUntrustedError {constructor() {super('Private-store inode changed')this.name = 'PrivateStoreNameMovedError'@@ -53,7 +65,7 @@ export async function openPrivateStoreObject(const identity = await handle.stat({ bigint: true })const expected = kind === 'directory' ? 0o700 : 0o600const actual = info.mode & 0o7777- if (!isPrivateStoreStatTrusted(info, kind)) throw new Error('Untrusted private-store inode')+ if (!isPrivateStoreStatTrusted(info, kind)) throw new PrivateStoreUntrustedError()if (actual !== expected && restore) {await beforeMutation?.()await handle.chmod(expected)@@ -67,7 +79,7 @@ export async function openPrivateStoreObject(named.isSymbolicLink() ||(after.mode & 0o7777) !== (restore ? expected : actual))- throw new Error('Private-store inode changed')+ throw new PrivateStoreUntrustedError('Private-store inode changed')if (named.dev !== identity.dev || named.ino !== identity.ino)throw new PrivateStoreNameMovedError()return handle