Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(mcp-host): keep failed duplicate removals charged across sweeps

Alfredo Lopez2a90cf735c52
diff --git a/docs/operations/gfs-large-file-host-downloads.md b/docs/operations/gfs-large-file-host-downloads.md
index 9f3933b7f..92cbb5aae 100644
--- a/docs/operations/gfs-large-file-host-downloads.md
+++ b/docs/operations/gfs-large-file-host-downloads.md
@@ -385,7 +385,13 @@ it` and counts `sweep_failed`. All three log the error code only.
A directory that cannot be listed is skipped and retried by the next sweep
(`sweep_failed`). A removal that fails is logged with its error code
(`remove_failed`) and retried by the next sweep; the copy stays where it is,
-indexed and charged, and a published copy stays published. Only a
+indexed and charged, and a published copy stays published. A duplicate
+directory whose removal failed stays charged against the Host budget until a
+removal succeeds or `lstat` answers `ENOENT` for it. When a later sweep cannot
+inspect it (an error other than a size, type or content mismatch) or cannot
+list its `.gfs-downloads`, the charge of the previous sweep carries over;
+a directory that is now indexed or being transferred is charged once, through
+its entry or reservation. Only a
removal that moved a directory counts as `incomplete_removed` or
`expired_removed`; one whose directory never existed counts nothing. Expiry is the boundary itself: a
copy whose `expiresAt` equals the current time is expired for reuse, managed
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index d0b4ad2b6..821971cb5 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.539",
+ "version": "0.1.540",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.539",
+ "version": "0.1.540",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index a6564b3b5..7198f029a 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.539",
+ "version": "0.1.540",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.ts
index ca3725f24..5d85d8b36 100644
--- a/mcp-host/src/internalTools/gfsDownloadStore.ts
+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts
@@ -1010,12 +1010,22 @@ export class GfsDownloadStore {
return result !== 'failed'
}
// A duplicate this sweep could not remove is still on disk, so it stays
- // charged: a failed cleanup never turns into free capacity.
+ // charged: a failed cleanup never turns into free capacity. Its charge is
+ // released only by a removal that succeeds or an lstat that answers
+ // ENOENT; an inspection or listing that fails carries the previous
+ // sweep's charge over instead of reading as zero.
+ const previousHeld = this.held
const held = new Map<string, HeldCopy>()
+ const settled = new Set<string>()
const removeDuplicate = async (id: string, directory: string): Promise<void> => {
+ settled.add(directory)
if (await removeIncomplete(directory)) return
const state = await this.inspectDirectory(directory, id, now)
if (state.state === 'complete') held.set(directory, { sizeBytes: state.entry.sizeBytes })
+ else if (state.state === 'unknown' || state.state === 'incomplete') {
+ const previous = previousHeld.get(directory)
+ if (previous) held.set(directory, previous)
+ }
}
for (const [id, directories] of candidates) {
@@ -1073,6 +1083,23 @@ export class GfsDownloadStore {
totals.retainedBytes += entry.sizeBytes
if (entry.provenance === 'adopted') totals.adopted += 1
}
+ // A held directory this sweep never reached (its listing failed, or it is
+ // no longer a duplicate) keeps its charge until lstat proves it gone.
+ for (const [directory, copy] of previousHeld) {
+ if (settled.has(directory)) continue
+ try {
+ await fs.lstat(directory)
+ } catch (error) {
+ if (errorCode(error) === 'ENOENT') continue
+ }
+ held.set(directory, copy)
+ }
+ // Never charged twice: a directory now indexed or being transferred is
+ // already counted through its entry or its reservation.
+ const counted = new Set<string>()
+ for (const entry of this.entries.values()) counted.add(entry.directory)
+ for (const transfer of this.active.values()) counted.add(transfer.directory)
+ for (const directory of held.keys()) if (counted.has(directory)) held.delete(directory)
this.held = held
return totals
}
diff --git a/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.ts
new file mode 100644
index 000000000..43d3be5bb
--- /dev/null
+++ b/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.ts
@@ -0,0 +1,251 @@
+/**
+ * R2-F2: a duplicate whose removal failed stays charged against the Host
+ * budget until its removal succeeds or `lstat` proves it gone, including
+ * through sweeps that cannot inspect or list it. Ported from the final
+ * review's `SOL-R2-HELD-meta-open` and `SOL-R2-HELD-caller-list` repros to the
+ * byte budget. Every case runs on a real temporary directory; faults are
+ * injected only on the operation that fails.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { randomUUID } from 'node:crypto'
+import * as syncFs from 'node:fs'
+import * as fs from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import * as path from 'node:path'
+import {
+ callerDirectory,
+ downloadDirectory,
+ exists,
+ metaFor,
+ plantDownload,
+ startTransfer,
+} from '../__tests__/fixtures/gfsStoreTestKit'
+import { GfsDownloadStore } from './gfsDownloadStore'
+
+type FsOperation = 'open' | 'lstat' | 'rename' | 'readdir'
+
+const { faults } = vi.hoisted(() => ({
+ faults: new Map<string, (target: string) => string | undefined>(),
+}))
+
+// A 95-byte volume at the default 85% gives a budget of floor(80.75) = 80:
+// eight 5-byte copies per caller fill it exactly.
+const VOLUME_BYTES = 95n
+
+vi.mock('node:fs/promises', async original => {
+ const actual = await original<typeof fs>()
+ const wrap =
+ <F extends (...args: never[]) => unknown>(operation: string, real: F) =>
+ (...args: Parameters<F>) => {
+ const code = faults.get(operation)?.(String(args[0]))
+ if (code !== undefined) {
+ const error = new Error(`${code}: injected`) as NodeJS.ErrnoException
+ error.code = code
+ return Promise.reject(error)
+ }
+ return real(...args)
+ }
+ return {
+ ...actual,
+ open: wrap('open', actual.open),
+ lstat: wrap('lstat', actual.lstat),
+ rename: wrap('rename', actual.rename),
+ readdir: wrap('readdir', actual.readdir),
+ statfs: async (target: string, options: { bigint: true }) => {
+ const real = await actual.statfs(target, options)
+ return { ...real, bsize: 1n, blocks: VOLUME_BYTES, bavail: real.bavail * real.bsize }
+ },
+ }
+})
+
+/** Records every `operation` call that `matches`, failing it with EIO. */
+function failWithEio(operation: FsOperation, matches: (target: string) => boolean): string[] {
+ const calls: string[] = []
+ faults.set(operation, target => {
+ if (!matches(target)) return undefined
+ calls.push(target)
+ return 'EIO'
+ })
+ return calls
+}
+
+const OWNER = 'caller-b'
+const PLANTER = 'caller-a'
+const DOWNLOADS = `${path.sep}users${path.sep}${PLANTER}${path.sep}.gfs-downloads`
+
+let hostRoot: string
+const stores: GfsDownloadStore[] = []
+
+async function openStore(): Promise<GfsDownloadStore> {
+ const opened = new GfsDownloadStore(hostRoot)
+ stores.push(opened)
+ await opened.initialize()
+ return opened
+}
+
+function plantCopies(root: string, ids: string[]): void {
+ const bytes = Buffer.from('quota')
+ for (const id of ids) plantDownload(root, { id, bytes, meta: metaFor(id, bytes, Date.now()) })
+}
+
+/** Renames of the given ids' input directories fail; `only` limits it to one caller root. */
+function failRemovals(ids: string[], only?: string): string[] {
+ const names = new Set(ids.map(id => `input-${id}`))
+ return failWithEio(
+ 'rename',
+ target => names.has(path.basename(target)) && (only === undefined || target.includes(only))
+ )
+}
+
+/**
+ * The owner's 8 copies are indexed at startup, the planter's 8 duplicates of
+ * the same ids appear afterwards, and every removal of either fails: 80 bytes
+ * on disk against a budget of 80, so a 5-byte admission must be refused.
+ */
+async function heldDuplicates() {
+ const owner = callerDirectory(hostRoot, OWNER)
+ const planter = callerDirectory(hostRoot, PLANTER)
+ const ids = Array.from({ length: 8 }, () => randomUUID())
+ plantCopies(owner, ids)
+ const store = await openStore()
+ const renames = failRemovals(ids)
+ plantCopies(planter, ids)
+ const first = await store.cleanupExpired()
+ // Witness: both copies of every id were removal attempts, and each failed.
+ expect(first.removeFailed).toBe(ids.length * 2)
+ expect(renames.length).toBeGreaterThanOrEqual(ids.length * 2)
+ await expect(startTransfer(store, planter, PLANTER, 100, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+ return { store, owner, planter, ids }
+}
+
+beforeEach(() => {
+ faults.clear()
+ hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-held-'))
+})
+
+afterEach(async () => {
+ faults.clear()
+ vi.restoreAllMocks()
+ for (const opened of stores.splice(0)) await opened.close(0)
+ await fs.rm(hostRoot, { recursive: true, force: true })
+})
+
+describe('GFS download store: held duplicate charges survive transient failures (R2-F2)', () => {
+ it('SOL-R2-HELD-meta-open: a duplicate whose meta.json cannot be opened keeps its charge', async () => {
+ const { store, planter, ids } = await heldDuplicates()
+ const reads = failWithEio(
+ 'open',
+ target => target.includes(`${DOWNLOADS}${path.sep}`) && path.basename(target) === 'meta.json'
+ )
+
+ // The sweep inside this admission inspects every duplicate and cannot.
+ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+ // Witness: the inspection was attempted and failed.
+ expect(reads.length).toBeGreaterThan(0)
+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)
+
+ // Inspection back, removals still failing: still refused.
+ faults.delete('open')
+ await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+
+ // Removals back: the sweep removes every duplicate and capacity returns.
+ faults.delete('rename')
+ await store.cleanupExpired()
+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)
+ const admitted = await startTransfer(store, planter, PLANTER, 103, 5)
+ await store.fail(admitted.transfer.id, PLANTER)
+ })
+
+ it('SOL-R2-HELD-caller-list: a duplicate whose .gfs-downloads cannot be listed keeps its charge', async () => {
+ const { store, planter, ids } = await heldDuplicates()
+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
+
+ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+ // Witness: the listing was attempted and failed.
+ expect(listings.length).toBeGreaterThan(0)
+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)
+
+ faults.delete('readdir')
+ await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+
+ faults.delete('rename')
+ await store.cleanupExpired()
+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)
+ const admitted = await startTransfer(store, planter, PLANTER, 103, 5)
+ await store.fail(admitted.transfer.id, PLANTER)
+ })
+
+ it('a carried charge is dropped once lstat reports the directory gone', async () => {
+ const { store, planter, ids } = await heldDuplicates()
+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
+ // The duplicates disappear while their parent cannot be listed.
+ for (const id of ids)
+ syncFs.rmSync(downloadDirectory(planter, id), { recursive: true, force: true })
+
+ // Only the owner's 40 bytes remain on disk: 40 + 5 fits the budget of 80.
+ const admitted = await startTransfer(store, planter, PLANTER, 101, 5)
+ // Witness: the sweep inside the admission could not list the duplicates,
+ // so only the lstat re-check can have released their charge.
+ expect(listings.length).toBeGreaterThan(0)
+ await store.fail(admitted.transfer.id, PLANTER)
+ })
+
+ it('a carried charge whose lstat fails with anything but ENOENT is kept', async () => {
+ const { store, planter, ids } = await heldDuplicates()
+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
+ const probes = failWithEio('lstat', target =>
+ ids.some(id => target.endsWith(`${DOWNLOADS}${path.sep}input-${id}`))
+ )
+
+ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
+ code: 'host_quota_exceeded',
+ })
+ // Witness: the re-check ran on the carried directories and failed.
+ expect(listings.length).toBeGreaterThan(0)
+ expect(probes.length).toBeGreaterThanOrEqual(ids.length)
+
+ faults.clear()
+ await store.cleanupExpired()
+ const admitted = await startTransfer(store, planter, PLANTER, 102, 5)
+ await store.fail(admitted.transfer.id, PLANTER)
+ })
+
+ it('a carried directory that is now indexed is charged once, not twice', async () => {
+ const owner = callerDirectory(hostRoot, OWNER)
+ const planter = callerDirectory(hostRoot, PLANTER)
+ const ids = Array.from({ length: 8 }, () => randomUUID())
+ plantCopies(owner, ids)
+ const store = await openStore()
+ // Only the planter's removals fail: the owner's indexed copies go.
+ const renames = failRemovals(ids, `${path.sep}${PLANTER}${path.sep}`)
+ plantCopies(planter, ids)
+
+ const first = await store.cleanupExpired()
+ expect(first.removeFailed).toBe(ids.length)
+ expect(renames.length).toBeGreaterThanOrEqual(ids.length)
+ for (const id of ids) expect(exists(downloadDirectory(owner, id))).toBe(false)
+
+ // The next sweep finds each planter copy alone and indexes it as adopted.
+ const second = await store.cleanupExpired()
+ expect(second.removeFailed).toBe(0)
+ expect((await store.debugInventory()).byCaller.get(PLANTER)).toEqual({ files: 8, bytes: 40 })
+
+ // 40 indexed + 40 requested = 80 fits the budget exactly; a second charge
+ // of the same 40 bytes would refuse it.
+ const renamesBefore = renames.length
+ const admitted = await startTransfer(store, planter, PLANTER, 101, 40)
+ // Witness: nothing had to be evicted to make room.
+ expect(renames.length).toBe(renamesBefore)
+ await store.fail(admitted.transfer.id, PLANTER)
+ })
+})
diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.json
index 357a97348..58bb874fc 100644
--- a/scripts/ci/pinned-vitest-cases/mcp-host.json
+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json
@@ -874,5 +874,25 @@
{
"suite": "gfsFilePolicy.test.ts",
"fullName": "GFS file policy removed retained-storage variables no longer shape the policy"
+ },
+ {
+ "suite": "gfsDownloadStoreHeldCharges.test.ts",
+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) SOL-R2-HELD-meta-open: a duplicate whose meta.json cannot be opened keeps its charge"
+ },
+ {
+ "suite": "gfsDownloadStoreHeldCharges.test.ts",
+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) SOL-R2-HELD-caller-list: a duplicate whose .gfs-downloads cannot be listed keeps its charge"
+ },
+ {
+ "suite": "gfsDownloadStoreHeldCharges.test.ts",
+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried charge is dropped once lstat reports the directory gone"
+ },
+ {
+ "suite": "gfsDownloadStoreHeldCharges.test.ts",
+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried charge whose lstat fails with anything but ENOENT is kept"
+ },
+ {
+ "suite": "gfsDownloadStoreHeldCharges.test.ts",
+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried directory that is now indexed is charged once, not twice"
}
]