Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): keep failed duplicate removals charged across sweeps
Alfredo Lopez2a90cf735c52
diff --git a/docs/operations/gfs-large-file-host-downloads.md b/docs/operations/gfs-large-file-host-downloads.mdindex 9f3933b7f..92cbb5aae 100644--- a/docs/operations/gfs-large-file-host-downloads.md+++ b/docs/operations/gfs-large-file-host-downloads.md@@ -385,7 +385,13 @@ it` and counts `sweep_failed`. All three log the error code only.A directory that cannot be listed is skipped and retried by the next sweep(`sweep_failed`). A removal that fails is logged with its error code(`remove_failed`) and retried by the next sweep; the copy stays where it is,-indexed and charged, and a published copy stays published. Only a+indexed and charged, and a published copy stays published. A duplicate+directory whose removal failed stays charged against the Host budget until a+removal succeeds or `lstat` answers `ENOENT` for it. When a later sweep cannot+inspect it (an error other than a size, type or content mismatch) or cannot+list its `.gfs-downloads`, the charge of the previous sweep carries over;+a directory that is now indexed or being transferred is charged once, through+its entry or reservation. Only aremoval that moved a directory counts as `incomplete_removed` or`expired_removed`; one whose directory never existed counts nothing. Expiry is the boundary itself: acopy whose `expiresAt` equals the current time is expired for reuse, manageddiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex d0b4ad2b6..821971cb5 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.539",+ "version": "0.1.540","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.539",+ "version": "0.1.540","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex a6564b3b5..7198f029a 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.539",+ "version": "0.1.540","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.tsindex ca3725f24..5d85d8b36 100644--- a/mcp-host/src/internalTools/gfsDownloadStore.ts+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts@@ -1010,12 +1010,22 @@ export class GfsDownloadStore {return result !== 'failed'}// A duplicate this sweep could not remove is still on disk, so it stays- // charged: a failed cleanup never turns into free capacity.+ // charged: a failed cleanup never turns into free capacity. Its charge is+ // released only by a removal that succeeds or an lstat that answers+ // ENOENT; an inspection or listing that fails carries the previous+ // sweep's charge over instead of reading as zero.+ const previousHeld = this.heldconst held = new Map<string, HeldCopy>()+ const settled = new Set<string>()const removeDuplicate = async (id: string, directory: string): Promise<void> => {+ settled.add(directory)if (await removeIncomplete(directory)) returnconst state = await this.inspectDirectory(directory, id, now)if (state.state === 'complete') held.set(directory, { sizeBytes: state.entry.sizeBytes })+ else if (state.state === 'unknown' || state.state === 'incomplete') {+ const previous = previousHeld.get(directory)+ if (previous) held.set(directory, previous)+ }}for (const [id, directories] of candidates) {@@ -1073,6 +1083,23 @@ export class GfsDownloadStore {totals.retainedBytes += entry.sizeBytesif (entry.provenance === 'adopted') totals.adopted += 1}+ // A held directory this sweep never reached (its listing failed, or it is+ // no longer a duplicate) keeps its charge until lstat proves it gone.+ for (const [directory, copy] of previousHeld) {+ if (settled.has(directory)) continue+ try {+ await fs.lstat(directory)+ } catch (error) {+ if (errorCode(error) === 'ENOENT') continue+ }+ held.set(directory, copy)+ }+ // Never charged twice: a directory now indexed or being transferred is+ // already counted through its entry or its reservation.+ const counted = new Set<string>()+ for (const entry of this.entries.values()) counted.add(entry.directory)+ for (const transfer of this.active.values()) counted.add(transfer.directory)+ for (const directory of held.keys()) if (counted.has(directory)) held.delete(directory)this.held = heldreturn totals}diff --git a/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.tsnew file mode 100644index 000000000..43d3be5bb--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreHeldCharges.test.ts@@ -0,0 +1,251 @@+/**+ * R2-F2: a duplicate whose removal failed stays charged against the Host+ * budget until its removal succeeds or `lstat` proves it gone, including+ * through sweeps that cannot inspect or list it. Ported from the final+ * review's `SOL-R2-HELD-meta-open` and `SOL-R2-HELD-caller-list` repros to the+ * byte budget. Every case runs on a real temporary directory; faults are+ * injected only on the operation that fails.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { randomUUID } from 'node:crypto'+import * as syncFs from 'node:fs'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import {+ callerDirectory,+ downloadDirectory,+ exists,+ metaFor,+ plantDownload,+ startTransfer,+} from '../__tests__/fixtures/gfsStoreTestKit'+import { GfsDownloadStore } from './gfsDownloadStore'++type FsOperation = 'open' | 'lstat' | 'rename' | 'readdir'++const { faults } = vi.hoisted(() => ({+ faults: new Map<string, (target: string) => string | undefined>(),+}))++// A 95-byte volume at the default 85% gives a budget of floor(80.75) = 80:+// eight 5-byte copies per caller fill it exactly.+const VOLUME_BYTES = 95n++vi.mock('node:fs/promises', async original => {+ const actual = await original<typeof fs>()+ const wrap =+ <F extends (...args: never[]) => unknown>(operation: string, real: F) =>+ (...args: Parameters<F>) => {+ const code = faults.get(operation)?.(String(args[0]))+ if (code !== undefined) {+ const error = new Error(`${code}: injected`) as NodeJS.ErrnoException+ error.code = code+ return Promise.reject(error)+ }+ return real(...args)+ }+ return {+ ...actual,+ open: wrap('open', actual.open),+ lstat: wrap('lstat', actual.lstat),+ rename: wrap('rename', actual.rename),+ readdir: wrap('readdir', actual.readdir),+ statfs: async (target: string, options: { bigint: true }) => {+ const real = await actual.statfs(target, options)+ return { ...real, bsize: 1n, blocks: VOLUME_BYTES, bavail: real.bavail * real.bsize }+ },+ }+})++/** Records every `operation` call that `matches`, failing it with EIO. */+function failWithEio(operation: FsOperation, matches: (target: string) => boolean): string[] {+ const calls: string[] = []+ faults.set(operation, target => {+ if (!matches(target)) return undefined+ calls.push(target)+ return 'EIO'+ })+ return calls+}++const OWNER = 'caller-b'+const PLANTER = 'caller-a'+const DOWNLOADS = `${path.sep}users${path.sep}${PLANTER}${path.sep}.gfs-downloads`++let hostRoot: string+const stores: GfsDownloadStore[] = []++async function openStore(): Promise<GfsDownloadStore> {+ const opened = new GfsDownloadStore(hostRoot)+ stores.push(opened)+ await opened.initialize()+ return opened+}++function plantCopies(root: string, ids: string[]): void {+ const bytes = Buffer.from('quota')+ for (const id of ids) plantDownload(root, { id, bytes, meta: metaFor(id, bytes, Date.now()) })+}++/** Renames of the given ids' input directories fail; `only` limits it to one caller root. */+function failRemovals(ids: string[], only?: string): string[] {+ const names = new Set(ids.map(id => `input-${id}`))+ return failWithEio(+ 'rename',+ target => names.has(path.basename(target)) && (only === undefined || target.includes(only))+ )+}++/**+ * The owner's 8 copies are indexed at startup, the planter's 8 duplicates of+ * the same ids appear afterwards, and every removal of either fails: 80 bytes+ * on disk against a budget of 80, so a 5-byte admission must be refused.+ */+async function heldDuplicates() {+ const owner = callerDirectory(hostRoot, OWNER)+ const planter = callerDirectory(hostRoot, PLANTER)+ const ids = Array.from({ length: 8 }, () => randomUUID())+ plantCopies(owner, ids)+ const store = await openStore()+ const renames = failRemovals(ids)+ plantCopies(planter, ids)+ const first = await store.cleanupExpired()+ // Witness: both copies of every id were removal attempts, and each failed.+ expect(first.removeFailed).toBe(ids.length * 2)+ expect(renames.length).toBeGreaterThanOrEqual(ids.length * 2)+ await expect(startTransfer(store, planter, PLANTER, 100, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ return { store, owner, planter, ids }+}++beforeEach(() => {+ faults.clear()+ hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-held-'))+})++afterEach(async () => {+ faults.clear()+ vi.restoreAllMocks()+ for (const opened of stores.splice(0)) await opened.close(0)+ await fs.rm(hostRoot, { recursive: true, force: true })+})++describe('GFS download store: held duplicate charges survive transient failures (R2-F2)', () => {+ it('SOL-R2-HELD-meta-open: a duplicate whose meta.json cannot be opened keeps its charge', async () => {+ const { store, planter, ids } = await heldDuplicates()+ const reads = failWithEio(+ 'open',+ target => target.includes(`${DOWNLOADS}${path.sep}`) && path.basename(target) === 'meta.json'+ )++ // The sweep inside this admission inspects every duplicate and cannot.+ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ // Witness: the inspection was attempted and failed.+ expect(reads.length).toBeGreaterThan(0)+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)++ // Inspection back, removals still failing: still refused.+ faults.delete('open')+ await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })++ // Removals back: the sweep removes every duplicate and capacity returns.+ faults.delete('rename')+ await store.cleanupExpired()+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)+ const admitted = await startTransfer(store, planter, PLANTER, 103, 5)+ await store.fail(admitted.transfer.id, PLANTER)+ })++ it('SOL-R2-HELD-caller-list: a duplicate whose .gfs-downloads cannot be listed keeps its charge', async () => {+ const { store, planter, ids } = await heldDuplicates()+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))++ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ // Witness: the listing was attempted and failed.+ expect(listings.length).toBeGreaterThan(0)+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)++ faults.delete('readdir')+ await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })++ faults.delete('rename')+ await store.cleanupExpired()+ for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)+ const admitted = await startTransfer(store, planter, PLANTER, 103, 5)+ await store.fail(admitted.transfer.id, PLANTER)+ })++ it('a carried charge is dropped once lstat reports the directory gone', async () => {+ const { store, planter, ids } = await heldDuplicates()+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))+ // The duplicates disappear while their parent cannot be listed.+ for (const id of ids)+ syncFs.rmSync(downloadDirectory(planter, id), { recursive: true, force: true })++ // Only the owner's 40 bytes remain on disk: 40 + 5 fits the budget of 80.+ const admitted = await startTransfer(store, planter, PLANTER, 101, 5)+ // Witness: the sweep inside the admission could not list the duplicates,+ // so only the lstat re-check can have released their charge.+ expect(listings.length).toBeGreaterThan(0)+ await store.fail(admitted.transfer.id, PLANTER)+ })++ it('a carried charge whose lstat fails with anything but ENOENT is kept', async () => {+ const { store, planter, ids } = await heldDuplicates()+ const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))+ const probes = failWithEio('lstat', target =>+ ids.some(id => target.endsWith(`${DOWNLOADS}${path.sep}input-${id}`))+ )++ await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ // Witness: the re-check ran on the carried directories and failed.+ expect(listings.length).toBeGreaterThan(0)+ expect(probes.length).toBeGreaterThanOrEqual(ids.length)++ faults.clear()+ await store.cleanupExpired()+ const admitted = await startTransfer(store, planter, PLANTER, 102, 5)+ await store.fail(admitted.transfer.id, PLANTER)+ })++ it('a carried directory that is now indexed is charged once, not twice', async () => {+ const owner = callerDirectory(hostRoot, OWNER)+ const planter = callerDirectory(hostRoot, PLANTER)+ const ids = Array.from({ length: 8 }, () => randomUUID())+ plantCopies(owner, ids)+ const store = await openStore()+ // Only the planter's removals fail: the owner's indexed copies go.+ const renames = failRemovals(ids, `${path.sep}${PLANTER}${path.sep}`)+ plantCopies(planter, ids)++ const first = await store.cleanupExpired()+ expect(first.removeFailed).toBe(ids.length)+ expect(renames.length).toBeGreaterThanOrEqual(ids.length)+ for (const id of ids) expect(exists(downloadDirectory(owner, id))).toBe(false)++ // The next sweep finds each planter copy alone and indexes it as adopted.+ const second = await store.cleanupExpired()+ expect(second.removeFailed).toBe(0)+ expect((await store.debugInventory()).byCaller.get(PLANTER)).toEqual({ files: 8, bytes: 40 })++ // 40 indexed + 40 requested = 80 fits the budget exactly; a second charge+ // of the same 40 bytes would refuse it.+ const renamesBefore = renames.length+ const admitted = await startTransfer(store, planter, PLANTER, 101, 40)+ // Witness: nothing had to be evicted to make room.+ expect(renames.length).toBe(renamesBefore)+ await store.fail(admitted.transfer.id, PLANTER)+ })+})diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonindex 357a97348..58bb874fc 100644--- a/scripts/ci/pinned-vitest-cases/mcp-host.json+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -874,5 +874,25 @@{"suite": "gfsFilePolicy.test.ts","fullName": "GFS file policy removed retained-storage variables no longer shape the policy"+ },+ {+ "suite": "gfsDownloadStoreHeldCharges.test.ts",+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) SOL-R2-HELD-meta-open: a duplicate whose meta.json cannot be opened keeps its charge"+ },+ {+ "suite": "gfsDownloadStoreHeldCharges.test.ts",+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) SOL-R2-HELD-caller-list: a duplicate whose .gfs-downloads cannot be listed keeps its charge"+ },+ {+ "suite": "gfsDownloadStoreHeldCharges.test.ts",+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried charge is dropped once lstat reports the directory gone"+ },+ {+ "suite": "gfsDownloadStoreHeldCharges.test.ts",+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried charge whose lstat fails with anything but ENOENT is kept"+ },+ {+ "suite": "gfsDownloadStoreHeldCharges.test.ts",+ "fullName": "GFS download store: held duplicate charges survive transient failures (R2-F2) a carried directory that is now indexed is charged once, not twice"}]