Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): keep GFS store copies on transient I/O errors

Alfredo Lopez539c3d2c92bf
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 3d058f119..850fed973 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.529",
+ "version": "0.1.530",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.529",
+ "version": "0.1.530",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index a1d6c7858..d0ad8c6a6 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.529",
+ "version": "0.1.530",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.ts
index ea0a60627..4e37bf417 100644
--- a/mcp-host/src/internalTools/gfsDownloadStore.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts
@@ -11,7 +11,11 @@ import {
GFS_HOST_ACTIVE_DOWNLOADS,
GFS_HOST_RETAINED_FILES,
} from './gfsFilePolicy'
-import { openPrivateStoreObject, verifyPrivateStoreDirectory } from './gfsStorePrivateFiles'
+import {
+ PrivateStoreUntrustedError,
+ openPrivateStoreObject,
+ verifyPrivateStoreDirectory,
+} from './gfsStorePrivateFiles'
const GFS_DOWNLOAD_STORE_LOG_COMPONENT = 'GfsDownloadStore'
@@ -157,9 +161,14 @@ export interface GfsDownloadInventory {
byCaller: Map<string, { bytes: number; files: number }>
}
+/**
+ * `unknown`: the disk did not answer (EMFILE, EIO, ...). Nothing is decided
+ * about the directory; it is kept as it is and inspected again later.
+ */
type DirectoryState =
| { state: 'absent' }
| { state: 'incomplete' }
+ | { state: 'unknown'; code: string }
| { state: 'complete'; entry: Entry }
function isValidSource(value: unknown): value is GfsImageSource {
@@ -239,6 +248,38 @@ function errorCode(error: unknown): string {
return typeof code === 'string' ? code : 'unknown'
}
+/**
+ * Errnos that are an answer about the object itself rather than about the
+ * process or the volume at this instant: it is gone, it is a symlink or sits
+ * under a non-directory, or its mode refuses the owner.
+ */
+const DEFINITIVE_ERRNO = new Set(['ENOENT', 'ELOOP', 'ENOTDIR', 'EACCES'])
+
+/**
+ * True only when a failure proves that a copy or directory is not what the
+ * store wrote: a size, inode or digest mismatch (raised as a store error), an
+ * untrusted owner, mode or type, or a definitive errno. Anything else (EMFILE,
+ * EIO, ENOMEM, ...) is transient: the object is kept and checked again later,
+ * never removed on its account.
+ */
+function isDefinitiveMismatch(error: unknown): boolean {
+ if (error instanceof GfsDownloadStoreError) return error.code !== 'publication_cancelled'
+ if (error instanceof PrivateStoreUntrustedError) return true
+ return DEFINITIVE_ERRNO.has(errorCode(error))
+}
+
+/** Errno, or the error class name for the store's own refusals; never a path. */
+function diagnosticCode(error: unknown): string {
+ const code = errorCode(error)
+ return code === 'unknown' && error instanceof Error ? error.name : code
+}
+
+function stateFor(error: unknown): DirectoryState {
+ return isDefinitiveMismatch(error)
+ ? { state: 'incomplete' }
+ : { state: 'unknown', code: errorCode(error) }
+}
+
/** A removal is proven by ENOENT, not by `fs.rm` returning. */
async function assertAbsent(target: string): Promise<void> {
try {
@@ -676,6 +717,7 @@ export class GfsDownloadStore {
)
if (!entry) return undefined
let verified = false
+ let transient: string | undefined
try {
const handle = await this.openEntryContent(entry)
try {
@@ -692,8 +734,18 @@ export class GfsDownloadStore {
} catch (error) {
if (error instanceof GfsDownloadStoreError && error.code === 'publication_cancelled')
throw error
+ if (!isDefinitiveMismatch(error)) transient = errorCode(error)
verified = false
}
+ if (transient !== undefined) {
+ // The disk did not answer: this is not evidence against the copy,
+ // which another task may still hold. It stays; this caller downloads.
+ logger.warn(
+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: transient },
+ 'GFS download store could not verify a published copy for reuse; it is kept and checked again later'
+ )
+ return undefined
+ }
if (!verified) {
await this.removeEntry(entry, 'incomplete_removed')
return undefined
@@ -749,7 +801,8 @@ export class GfsDownloadStore {
} finally {
await handle.close()
}
- } catch {
+ } catch (error) {
+ this.reportManagedReadFailure(error)
throw new GfsDownloadStoreError('download_missing')
}
if (createHash('sha256').update(bytes).digest('hex') !== entry.sha256) {
@@ -781,7 +834,8 @@ export class GfsDownloadStore {
} finally {
await handle.close()
}
- } catch {
+ } catch (error) {
+ this.reportManagedReadFailure(error)
throw new GfsDownloadStoreError('download_missing')
}
}
@@ -927,6 +981,15 @@ export class GfsDownloadStore {
this.forget(id)
continue
}
+ if (state.state === 'unknown') {
+ // Kept as it is, indexed or not; the next sweep inspects it again.
+ recordGfsDownloadExpiry('sweep_failed')
+ logger.warn(
+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: state.code },
+ 'GFS download store could not inspect a download directory; it is kept and the next sweep retries it'
+ )
+ continue
+ }
if (state.state === 'incomplete') {
this.forget(id)
await removeIncomplete(directory)
@@ -1088,7 +1151,7 @@ export class GfsDownloadStore {
info = await fs.lstat(directory)
} catch (error) {
if (errorCode(error) === 'ENOENT') return { state: 'absent' }
- return { state: 'incomplete' }
+ return stateFor(error)
}
if (!info.isDirectory()) return { state: 'incomplete' }
let raw: string
@@ -1104,16 +1167,16 @@ export class GfsDownloadStore {
} finally {
await handle.close()
}
- } catch {
- return { state: 'incomplete' }
+ } catch (error) {
+ return stateFor(error)
}
const meta = parseMeta(raw, id, now)
if (!meta) return { state: 'incomplete' }
try {
const source = await fs.lstat(path.join(directory, SOURCE_FILE))
if (!source.isFile() || source.size !== meta.sizeBytes) return { state: 'incomplete' }
- } catch {
- return { state: 'incomplete' }
+ } catch (error) {
+ return stateFor(error)
}
return {
state: 'complete',
@@ -1127,19 +1190,23 @@ export class GfsDownloadStore {
}
}
- /** An indexed entry is re-checked with one lstat; meta.json is not re-read. */
+ /**
+ * An indexed entry is re-checked with one lstat; meta.json is not re-read.
+ * Only a definitive answer makes it incomplete; a transient errno keeps it.
+ */
private async recheckIndexed(entry: Entry): Promise<DirectoryState> {
try {
const source = await fs.lstat(path.join(entry.directory, SOURCE_FILE))
if (source.isFile() && source.size === entry.sizeBytes) return { state: 'complete', entry }
+ return { state: 'incomplete' }
} catch (error) {
- if (errorCode(error) === 'ENOENT') {
- try {
- await fs.lstat(entry.directory)
- } catch (directoryError) {
- if (errorCode(directoryError) === 'ENOENT') return { state: 'absent' }
- }
- }
+ if (errorCode(error) !== 'ENOENT') return stateFor(error)
+ }
+ try {
+ await fs.lstat(entry.directory)
+ } catch (directoryError) {
+ if (errorCode(directoryError) === 'ENOENT') return { state: 'absent' }
+ return stateFor(directoryError)
}
return { state: 'incomplete' }
}
@@ -1206,9 +1273,13 @@ export class GfsDownloadStore {
}
/**
- * A `.gfs-downloads` that is a symlink, not a directory, or not private to
- * this process is removed without following it and recreated, so a shell
- * command can cost a caller its cached copies but never block its downloads.
+ * A `.gfs-downloads` proven not to be the store's (a symlink, not a
+ * directory, gone, unreadable to its owner, or not owned by this process
+ * with a private mode) is removed without following it and recreated, so a
+ * shell command can cost a caller its cached copies but never block its
+ * downloads. Any other failure to verify it (EMFILE, EIO, ...) proves
+ * nothing: it is rethrown, the admission fails and the directory and its
+ * copies stay.
*/
private async ensureDownloadsRoot(downloadsRoot: string): Promise<void> {
try {
@@ -1220,8 +1291,8 @@ export class GfsDownloadStore {
try {
await verifyPrivateStoreDirectory(downloadsRoot)
return
- } catch {
- // Not a private directory: replaced below.
+ } catch (error) {
+ if (!isDefinitiveMismatch(error)) throw error
}
for (const id of this.idsUnder(downloadsRoot)) this.forget(id)
await this.removeVerified(downloadsRoot)
@@ -1406,6 +1477,20 @@ export class GfsDownloadStore {
if (real !== parent) throw new GfsDownloadStoreError('workspace_unavailable')
}
+ /**
+ * The caller always gets download_missing, so a read failure reveals
+ * nothing about the copy. The owner's own published copy failing for any
+ * reason but its absence is worth an operator's attention; code only.
+ */
+ private reportManagedReadFailure(error: unknown): void {
+ if (errorCode(error) === 'ENOENT') return
+ if (error instanceof GfsDownloadStoreError && error.code === 'download_missing') return
+ logger.warn(
+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: diagnosticCode(error) },
+ 'GFS download store could not read a published copy for its caller'
+ )
+ }
+
private reportSweepFailure(error: unknown): void {
recordGfsDownloadExpiry('sweep_failed')
logger.warn(
diff --git a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts
index be0635e7f..5eb91e4a1 100644
--- a/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStoreAdversarial.test.ts
@@ -10,7 +10,17 @@ import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
-import { callerDirectory, completedCopy, sourceFor } from '../__tests__/fixtures/gfsStoreTestKit'
+import {
+ callerDirectory,
+ completedCopy,
+ exists,
+ expiryCount,
+ metaFor,
+ plantDownload,
+ sourceFor,
+ startTransfer,
+} from '../__tests__/fixtures/gfsStoreTestKit'
+import { logger } from '../logger'
import { deriveUserKey } from '../workspace/userKey'
import { GfsDownloadStore } from './gfsDownloadStore'
@@ -51,6 +61,36 @@ function injectFault(operation: FsOperation, fault: Fault): void {
faults.set(operation, fault)
}
+/**
+ * Watches `operation` on every path ending with `suffix`: records each call
+ * and fails it with `code`, or only records it when `code` is undefined.
+ */
+function watch(operation: FsOperation, suffix: string, code?: string): string[] {
+ const calls: string[] = []
+ injectFault(operation, target => {
+ if (!target.endsWith(suffix)) return undefined
+ calls.push(target)
+ return code
+ })
+ return calls
+}
+
+/** Every warning logged, as `{ code, message }`. */
+function warnings(warn: {
+ mock: { calls: unknown[][] }
+}): Array<{ code: unknown; message: unknown }> {
+ return warn.mock.calls.map(call => ({
+ code: (call[0] as { code?: unknown }).code,
+ message: call[1],
+ }))
+}
+
+const REUSE_KEPT =
+ 'GFS download store could not verify a published copy for reuse; it is kept and checked again later'
+const SWEEP_KEPT =
+ 'GFS download store could not inspect a download directory; it is kept and the next sweep retries it'
+const READ_FAILED = 'GFS download store could not read a published copy for its caller'
+
let hostRoot: string
const stores: GfsDownloadStore[] = []
@@ -164,3 +204,130 @@ describe('GFS download store adversarial round: caller root shape (F9)', () => {
).resolves.toMatchObject({ sizeBytes: 8 })
})
})
+
+describe('GFS download store adversarial round: transient I/O is not corruption (F1/ADV-3)', () => {
+ const KEY = 'alice-key'
+
+ it('reuse that cannot open a pinned copy keeps it for the task that holds it', async () => {
+ const store = await openStore()
+ const root = callerDirectory(hostRoot, KEY)
+ const { receipt, bytes } = await completedCopy(store, root, KEY, 1, 128, { owner: 'task-1' })
+ const warn = vi.spyOn(logger, 'warn')
+ const opens = watch('open', `input-${receipt.id}/source`, 'EMFILE')
+
+ await expect(
+ store.reusableReceipt(KEY, sourceFor(1), 128, { retentionOwnerId: 'task-2' })
+ ).resolves.toBeUndefined()
+ faults.clear()
+
+ // Witness: reuse reached the copy and failed on it.
+ expect(opens.length).toBeGreaterThan(0)
+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: REUSE_KEPT })
+ expect(exists(path.join(root, receipt.path))).toBe(true)
+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)
+ // Once the disk answers, the same copy is reused.
+ await expect(store.reusableReceipt(KEY, sourceFor(1), 128)).resolves.toMatchObject({
+ id: receipt.id,
+ })
+ })
+
+ it('reuse still removes a copy whose bytes no longer match its receipt', async () => {
+ const store = await openStore()
+ const root = callerDirectory(hostRoot, KEY)
+ const { receipt } = await completedCopy(store, root, KEY, 2, 64)
+ const removed = await expiryCount('incomplete_removed')
+ syncFs.writeFileSync(path.join(root, receipt.path), Buffer.alloc(64, 0xee))
+
+ await expect(store.reusableReceipt(KEY, sourceFor(2), 64)).resolves.toBeUndefined()
+ expect(exists(path.join(root, `.gfs-downloads/input-${receipt.id}`))).toBe(false)
+ expect(await expiryCount('incomplete_removed')).toBe(removed + 1)
+ })
+
+ it('a sweep that cannot lstat a published copy keeps it and counts sweep_failed', async () => {
+ const store = await openStore()
+ const root = callerDirectory(hostRoot, KEY)
+ const { receipt, bytes } = await completedCopy(store, root, KEY, 3, 96, { owner: 'task-1' })
+ const swept = await expiryCount('sweep_failed')
+ const warn = vi.spyOn(logger, 'warn')
+ const lstats = watch('lstat', `input-${receipt.id}/source`, 'EMFILE')
+
+ const result = await store.cleanupExpired()
+ faults.clear()
+
+ expect(lstats.length).toBeGreaterThan(0)
+ expect(result).toMatchObject({ removedIncomplete: 0, removedExpired: 0, removeFailed: 0 })
+ expect(await expiryCount('sweep_failed')).toBe(swept + 1)
+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: SWEEP_KEPT })
+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)
+ // Contrast: a source that is really gone is removed by the next sweep.
+ syncFs.rmSync(path.join(root, receipt.path))
+ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })
+ })
+
+ it('a startup sweep that cannot open meta.json keeps the copy for the next start', async () => {
+ const root = callerDirectory(hostRoot, KEY)
+ const bytes = Buffer.alloc(100, 7)
+ const id = '00000000-0000-4000-8000-000000000001'
+ const planted = plantDownload(root, { id, bytes, meta: metaFor(id, bytes, Date.now()) })
+ const opens = watch('open', '/meta.json', 'EMFILE')
+
+ await openStore()
+ faults.clear()
+
+ expect(opens.length).toBeGreaterThan(0)
+ expect(exists(planted.directory)).toBe(true)
+ // Witness: a start that can read it adopts it.
+ const second = await openStore()
+ expect((await second.debugInventory()).byCaller.get(KEY)).toEqual({ files: 1, bytes: 100 })
+ })
+
+ it('an admission that cannot verify .gfs-downloads fails and keeps every copy', async () => {
+ const store = await openStore()
+ const root = callerDirectory(hostRoot, KEY)
+ const { receipt, bytes } = await completedCopy(store, root, KEY, 4, 80, { owner: 'task-1' })
+ const opens = watch('open', `${KEY}/.gfs-downloads`, 'EMFILE')
+
+ await expect(startTransfer(store, root, KEY, 5, 40)).rejects.toMatchObject({
+ code: 'storage_write_failed',
+ })
+ faults.clear()
+
+ expect(opens.length).toBeGreaterThan(0)
+ await expect(store.readManagedFile(receipt.path, KEY)).resolves.toEqual(bytes)
+ // Contrast: a `.gfs-downloads` with a non-private mode is replaced.
+ syncFs.chmodSync(path.join(root, '.gfs-downloads'), 0o755)
+ const replaced = await startTransfer(store, root, KEY, 6, 40)
+ expect(exists(path.join(root, receipt.path))).toBe(false)
+ expect(exists(path.join(root, replaced.transfer.partialPath))).toBe(true)
+ })
+
+ it.each([
+ ['readManagedFile', (store: GfsDownloadStore, p: string) => store.readManagedFile(p, KEY)],
+ [
+ 'readManagedFilePrefix',
+ (store: GfsDownloadStore, p: string) => store.readManagedFilePrefix(p, KEY),
+ ],
+ ])(
+ '%s logs a non-ENOENT failure of the owner’s copy and still answers download_missing',
+ async (_name, read) => {
+ const store = await openStore()
+ const root = callerDirectory(hostRoot, KEY)
+ const { receipt } = await completedCopy(store, root, KEY, 7, 32)
+ const warn = vi.spyOn(logger, 'warn')
+ const failed = watch('open', `input-${receipt.id}/source`, 'EMFILE')
+
+ await expect(read(store, receipt.path)).rejects.toMatchObject({ code: 'download_missing' })
+ expect(failed).toHaveLength(1)
+ expect(warnings(warn)).toContainEqual({ code: 'EMFILE', message: READ_FAILED })
+
+ // ENOENT is the copy's absence: the same answer, nothing logged. The
+ // witness is the open the read had to attempt.
+ warn.mockClear()
+ syncFs.rmSync(path.join(root, receipt.path))
+ const attempted = watch('open', `input-${receipt.id}/source`)
+ await expect(read(store, receipt.path)).rejects.toMatchObject({ code: 'download_missing' })
+ expect(attempted).toHaveLength(1)
+ expect(warnings(warn).map(entry => entry.message)).not.toContain(READ_FAILED)
+ }
+ )
+})
diff --git a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts b/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
index c3e948c3a..a79a656ad 100644
--- a/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
+++ b/mcp-host/src/internalTools/gfsStorePrivateFiles.ts
@@ -22,11 +22,23 @@ export function isPrivateStoreStatTrusted(info: Stats, kind: 'file' | 'directory
)
}
+/**
+ * The object was opened and proved not to be the store's: wrong owner, mode,
+ * type or link count, or it changed while it was checked. Unlike an errno
+ * such as EMFILE or EIO, this is a definitive answer about the object.
+ */
+export class PrivateStoreUntrustedError extends Error {
+ constructor(message = 'Untrusted private-store inode') {
+ super(message)
+ this.name = 'PrivateStoreUntrustedError'
+ }
+}
+
/**
* The opened inode is trusted, but the name now points at another inode. An
* atomic rename over the name between open and lstat produces exactly this.
*/
-export class PrivateStoreNameMovedError extends Error {
+export class PrivateStoreNameMovedError extends PrivateStoreUntrustedError {
constructor() {
super('Private-store inode changed')
this.name = 'PrivateStoreNameMovedError'
@@ -53,7 +65,7 @@ export async function openPrivateStoreObject(
const identity = await handle.stat({ bigint: true })
const expected = kind === 'directory' ? 0o700 : 0o600
const actual = info.mode & 0o7777
- if (!isPrivateStoreStatTrusted(info, kind)) throw new Error('Untrusted private-store inode')
+ if (!isPrivateStoreStatTrusted(info, kind)) throw new PrivateStoreUntrustedError()
if (actual !== expected && restore) {
await beforeMutation?.()
await handle.chmod(expected)
@@ -67,7 +79,7 @@ export async function openPrivateStoreObject(
named.isSymbolicLink() ||
(after.mode & 0o7777) !== (restore ? expected : actual)
)
- throw new Error('Private-store inode changed')
+ throw new PrivateStoreUntrustedError('Private-store inode changed')
if (named.dev !== identity.dev || named.ino !== identity.ino)
throw new PrivateStoreNameMovedError()
return handle