Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(mcp-host): keep bytes charged whenever a GFS removal leaves them on disk
Alfredo Lopez7ea0d4197d45
diff --git a/docs/operations/gfs-large-file-host-downloads.md b/docs/operations/gfs-large-file-host-downloads.mdindex 407265903..a23d6a088 100644--- a/docs/operations/gfs-large-file-host-downloads.md+++ b/docs/operations/gfs-large-file-host-downloads.md@@ -448,10 +448,20 @@ removal keeps failing stays charged until it is removed. An indexed entry leftunder a trash name is dropped from the index, because its name is gone, and ischarged only on the trash name, never twice. An unindexed expired copy whoseremoval fails after the rename is charged on its trash name with the size read-from its `meta.json`. When the parent-check refuses and the rename back also fails, the refusal is reported and the-undo failure is logged as `GFS download store could not restore a directory-after refusing to remove it` with its error code.+from its `meta.json`. A directory the store holds no charge for when it+removes it (an unpublished transfer's partial, an unindexed incomplete+directory, or a trash directory left before this process started, since+charges are held in memory) is measured on its trash name (the bytes of its+regular files, symlinks not followed) and charged that size if its removal+fails. A tree that cannot be measured is logged and stays uncharged until a+later sweep measures it. When the rename itself fails, the directory keeps its+name: an indexed entry keeps its charge, and a directory with no charge is+measured the same way and charged under its own name. When the parent check+refuses after the rename and the rename back also fails, the refusal is+reported, the undo failure is logged as `GFS download store could not restore a+directory after refusing to remove it` with its error code, and a charge in+hand moves to the trash name; a trash directory in a refused parent is not+read to measure it.### Retention pins and cold resume@@ -520,10 +530,11 @@ user what happened and what they can do:frees as tasks finish and downloaded copies expire. The model tells the user,who can finish or cancel their own running tasks to free space sooner andthen try again. It never acts on another user's tasks or files. The text- offers no deletion of downloaded copies: the store has already evicted every- copy no running task protects before refusing, and the per-caller cap counts- only the caller's protected copies, so deleting copies by hand cannot admit- the download.+ offers no deletion of downloaded copies, because that cannot admit the+ download: the budget refusal comes only when evicting every copy no running+ task protects would still not make room (the store then evicts nothing), and+ the per-caller cap is checked before any eviction and counts only the copies+ the caller's running tasks protect.`volume_unmeasurable` carries no guidance: the volume could not be measured,and no cleanup by the user would change that. `limit_exceeded` and every otherdiff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex e0fc4d269..246e267a7 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.545",+ "version": "0.1.546","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.545",+ "version": "0.1.546","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex 1c091db4e..3e069b976 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.545",+ "version": "0.1.546","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/internalTools/gfsDownloadStore.ts b/mcp-host/src/internalTools/gfsDownloadStore.tsindex 47e2d1dbe..193a36533 100644--- a/mcp-host/src/internalTools/gfsDownloadStore.ts+++ b/mcp-host/src/internalTools/gfsDownloadStore.ts@@ -209,14 +209,18 @@ type RemovalResult = 'removed' | 'absent' | 'failed' | 'trashed'/*** A removal that renamed `directory` to `trash` and then could not remove* `trash` (or could not prove it gone). `code` is the underlying errno or- * store code, so logs and errorCode() see the real cause.+ * store code, so logs and errorCode() see the real cause. `refusal` is set+ * when the parent check refused `trash` and the rename back failed: the+ * bytes are under `trash`, the refusal is what callers report, and `trash`+ * is not measured because its parent did not pass the check.*/class TrashLeftError extends Error {readonly code: stringconstructor(readonly trash: string,- cause: unknown+ cause: unknown,+ readonly refusal?: unknown) {super('GFS download store left a renamed directory on disk')this.name = 'TrashLeftError'@@ -224,6 +228,20 @@ class TrashLeftError extends Error {}}+/**+ * A removal whose rename failed after the parent check passed: the directory+ * and its bytes are still under their own name.+ */+class KeptInPlaceError extends Error {+ readonly code: string++ constructor(cause: unknown) {+ super('GFS download store could not rename a directory it is removing')+ this.name = 'KeptInPlaceError'+ this.code = errorCode(cause)+ }+}+/*** `unknown`: the disk did not answer (EMFILE, EIO, ...). Nothing is decided* about the directory; it is kept as it is and inspected again later.@@ -1284,7 +1302,9 @@ export class GfsDownloadStore {return found}// A trash directory a held charge names keeps that charge through the- // removal: removeDirectory moves it to the next trash name or releases it.+ // removal: removeDirectory keeps it, moves it to the next trash name, or+ // releases it, and measures one with no held charge (left before this+ // process started) when its removal fails.const removeStray = async (directory: string, sweep: SweepTotals) => {const result = await this.removeDirectory(directory, 'incomplete_removed')if (result === 'removed') sweep.removedIncomplete += 1@@ -1320,6 +1340,23 @@ export class GfsDownloadStore {return found}+ /**+ * Bytes of regular files under a directory the store is removing and holds+ * no charge for, without following symlinks. A tree that cannot be+ * measured is logged with its code and left uncharged.+ */+ private async sizeStray(directory: string): Promise<number | undefined> {+ try {+ return (await treeUsage(directory)).bytes+ } catch (error) {+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: errorCode(error) },+ 'GFS download store could not measure a directory it is removing; it is not charged until a sweep measures it'+ )+ return undefined+ }+ }+/*** A replaced `.gfs-downloads` is renamed to `.gfs-downloads.trash-<uuid>`* in its caller root; one a stopped removal left there still holds copies@@ -1504,8 +1541,10 @@ export class GfsDownloadStore {} catch (error) {// Renamed but not removed: every copy charged under the old name is// now on disk under the trash name, so the charge moves there.- if (error instanceof TrashLeftError)+ if (error instanceof TrashLeftError) {this.moveChargesToTrash(downloadsRoot, replaced, error.trash)+ if (error.refusal !== undefined) throw error.refusal+ }throw error}for (const id of replaced) this.forget(id)@@ -1703,7 +1742,8 @@ export class GfsDownloadStore {* Retained bytes stay charged while they are on disk. The charge is* `charge` when the caller gives one (an indexed entry's size), otherwise a* held charge already on `directory` (a held duplicate or a trash directory- * an earlier removal left). When the rename to the trash name succeeds but+ * an earlier removal left), otherwise the measured size of the trash. When+ * the rename to the trash name succeeds but* the trash is not removed, the charge moves to the trash name as a held* charge, so the old name's absence never releases it; it is released only* when a later removal of that trash succeeds or lstat proves it gone@@ -1724,9 +1764,24 @@ export class GfsDownloadStore {{ component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: errorCode(error) },'GFS download store could not remove a download directory; the next sweep retries it')+ if (error instanceof KeptInPlaceError) {+ // Still under its own name. With no charge in hand (a stray trash, an+ // unindexed partial, a failed transfer) it is measured and held here;+ // a caller's charge stays with its indexed entry.+ if (carried === undefined) {+ const kept = await this.sizeStray(directory)+ if (kept !== undefined) this.held.set(directory, { sizeBytes: kept })+ }+ return 'failed'+ }if (!(error instanceof TrashLeftError)) return 'failed'this.held.delete(directory)- if (carried !== undefined) this.held.set(error.trash, { sizeBytes: carried })+ // No charge in hand: the bytes the rename moved are measured on the+ // trash name when it passed the parent check. One the check refused is+ // not read; only a charge in hand follows it there.+ const moved =+ carried ?? (error.refusal === undefined ? await this.sizeStray(error.trash) : undefined)+ if (moved !== undefined) this.held.set(error.trash, { sizeBytes: moved })return 'trashed'}this.held.delete(directory)@@ -1754,7 +1809,7 @@ export class GfsDownloadStore {try {await fs.rename(directory, trash)} catch (error) {- if (errorCode(error) !== 'ENOENT') throw error+ if (errorCode(error) !== 'ENOENT') throw new KeptInPlaceError(error)await assertAbsent(directory)return 'absent'}@@ -1764,12 +1819,13 @@ export class GfsDownloadStore {try {await fs.rename(trash, directory)} catch (undoError) {- // The refusal is what the caller must see; the trash name it is left- // under is one the sweep collects.+ // The refusal is what the caller must see; the bytes are left under+ // the trash name, which the sweep collects, so the charge moves there.logger.warn({ component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: errorCode(undoError) },'GFS download store could not restore a directory after refusing to remove it')+ throw new TrashLeftError(trash, error, error)}throw error}diff --git a/mcp-host/src/internalTools/gfsDownloadStoreRemovalCharge.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreRemovalCharge.test.tsnew file mode 100644index 000000000..47be47d0b--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreRemovalCharge.test.ts@@ -0,0 +1,595 @@+/**+ * Removal-charge accounting of the GFS download store.+ *+ * Whenever a removal leaves bytes on disk (in place, or under a `.trash-` name), those bytes stay+ * charged against the Host budget until the directory is removed or confirmed absent. Each test+ * runs on a coherent simulated volume (1000 MiB at 1%, so a 10 MiB budget) whose free space is the+ * source bytes actually present, injects one filesystem fault, asserts the refusal, then clears the+ * fault and proves the store recovers with a fresh publish and read.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as syncFs from 'node:fs'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import {+ callerDirectory,+ completedCopy,+ digestOf,+ downloadDirectory,+ quotaCount,+ sourceFor,+ startTransfer,+ withEnvironment,+} from '../__tests__/fixtures/gfsStoreTestKit'+import type { GfsDownloadStore } from './gfsDownloadStore'++const boundaries = vi.hoisted(() => ({+ open: vi.fn(),+ statfs: vi.fn(),+ rm: vi.fn(),+ rename: vi.fn(),+ realpath: vi.fn(),+ lstat: vi.fn(),+ readdir: vi.fn(),+}))+vi.mock('node:fs/promises', async original => ({+ ...(await original<typeof fs>()),+ ...boundaries,+}))++const MIB = 1024 * 1024+const HOUR = 60 * 60_000+const TOTAL = 1000n * BigInt(MIB)+let nativeFs: typeof fs+let hostRoot: string+let roots: Record<string, string>+const stores: GfsDownloadStore[] = []+const releases: Array<() => void> = []+const eio = () => Object.assign(new Error('EIO: injected fault'), { code: 'EIO' })++function storedBytes(directory: string): number {+ let sum = 0+ for (const child of syncFs.readdirSync(directory, { withFileTypes: true })) {+ const target = path.join(directory, child.name)+ if (child.isDirectory()) sum += storedBytes(target)+ else if (child.isFile() && ['source', 'source.partial'].includes(child.name))+ sum += syncFs.statSync(target).size+ }+ return sum+}++function view(store: GfsDownloadStore) {+ const state = store as unknown as {+ entries: Map<string, { sizeBytes: number }>+ active: Map<string, { sizeBytes: number }>+ held: Map<string, { sizeBytes: number }>+ pins: Map<string, Set<string>>+ }+ const sum = (map: Map<string, { sizeBytes: number }>) =>+ [...map.values()].reduce((n, item) => n + item.sizeBytes, 0)+ return {+ state,+ entries: sum(state.entries),+ active: sum(state.active),+ held: sum(state.held),+ charged: sum(state.entries) + sum(state.active) + sum(state.held),+ }+}++function restoreBoundary(name: keyof typeof boundaries): void {+ boundaries[name].mockReset()+ boundaries[name].mockImplementation((...args: unknown[]) =>+ Reflect.apply(nativeFs[name] as (...args: unknown[]) => unknown, nativeFs, args)+ )+}++async function storeWithBudget(): Promise<GfsDownloadStore> {+ return withEnvironment({ MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT: '1' }, async () => {+ vi.resetModules()+ const { GfsDownloadStore: Store } = await import('./gfsDownloadStore')+ const store = new Store(hostRoot) as GfsDownloadStore+ stores.push(store)+ await store.initialize()+ return store+ })+}++function failTrashRemoval(): string[] {+ const hits: string[] = []+ boundaries.rm.mockImplementation((...args: unknown[]) => {+ if (path.basename(String(args[0])).startsWith('.trash-')) {+ hits.push(String(args[0]))+ return Promise.reject(eio())+ }+ return Reflect.apply(nativeFs.rm, nativeFs, args)+ })+ return hits+}++async function expiredTrash(store: GfsDownloadStore) {+ await completedCopy(store, roots.a, 'a', 1, 4 * MIB, { owner: 'owner-a' })+ await completedCopy(store, roots.b, 'b', 2, 4 * MIB, { owner: 'owner-b' })+ vi.setSystemTime(Date.now() + 2 * HOUR)+ const hits = failTrashRemoval()+ expect((await store.cleanupExpired()).removeFailed).toBe(2)+ expect(hits).toHaveLength(2)+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ expect(view(store)).toMatchObject({ entries: 0, active: 0, held: 8 * MIB, charged: 8 * MIB })+ return hits+}++/** 8 MiB stay on disk, so a third 4 MiB admission would exceed the 10 MiB budget. */+async function expectRefused(store: GfsDownloadStore): Promise<void> {+ await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+}++/** Clears every fault, lets cleanup remove the leftovers, then publishes and reads a fresh copy. */+async function recoverAndWitness(store: GfsDownloadStore): Promise<void> {+ for (const name of ['rm', 'rename', 'realpath', 'open'] as const) restoreBoundary(name)+ await store.cleanupExpired()+ expect(storedBytes(hostRoot)).toBe(0)+ const fresh = await completedCopy(store, roots.c, 'c', 3, 4 * MIB, { owner: 'fresh' })+ expect((await store.readManagedFile(fresh.receipt.path, 'c')).equals(fresh.bytes)).toBe(true)+ expect(view(store).charged).toBe(4 * MIB)+}++beforeEach(async () => {+ nativeFs = await vi.importActual<typeof fs>('node:fs/promises')+ for (const name of Object.keys(boundaries) as Array<keyof typeof boundaries>)+ restoreBoundary(name)+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-08T12:00:00Z'))+ hostRoot = await nativeFs.realpath(syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-removal-charge-')))+ roots = Object.fromEntries(['a', 'b', 'c', 'd'].map(id => [id, callerDirectory(hostRoot, id)]))+ boundaries.statfs.mockImplementation(async (target: string) => ({+ ...(await nativeFs.statfs(target, { bigint: true })),+ bsize: 1n,+ blocks: TOTAL,+ bavail: TOTAL - BigInt(storedBytes(hostRoot)),+ }))+})++afterEach(async () => {+ for (const release of releases.splice(0)) release()+ vi.restoreAllMocks()+ vi.useRealTimers()+ for (const store of stores.splice(0)) await store.close(0)+ await nativeFs.rm(hostRoot, { recursive: true, force: true })+})++describe('GFS download store removal charges', () => {+ it('RC-UNDO-FAIL: post-rename verification and undo failures retain the renamed bytes', async () => {+ const store = await storeWithBudget()+ await completedCopy(store, roots.a, 'a', 1, 4 * MIB, { owner: 'a-owner' })+ await completedCopy(store, roots.b, 'b', 2, 4 * MIB, { owner: 'b-owner' })+ vi.setSystemTime(Date.now() + 2 * HOUR)+ const checkFault = new Set<string>()+ const undone: string[] = []+ boundaries.rename.mockImplementation(async (...args: unknown[]) => {+ const from = String(args[0])+ const to = String(args[1])+ if (path.basename(from).startsWith('.trash-') && path.basename(to).startsWith('input-')) {+ undone.push(from)+ throw eio()+ }+ const result = await Reflect.apply(nativeFs.rename, nativeFs, args)+ if (path.basename(from).startsWith('input-') && path.basename(to).startsWith('.trash-'))+ checkFault.add(path.dirname(to))+ return result+ })+ boundaries.realpath.mockImplementation(async (...args: unknown[]) => {+ if (checkFault.delete(String(args[0]))) throw eio()+ return Reflect.apply(nativeFs.realpath, nativeFs, args)+ })+ failTrashRemoval()+ expect((await store.cleanupExpired()).removeFailed).toBe(2)+ expect(undone).toHaveLength(2)+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ // The carried charges moved to the trash names in this sweep, not in a later one.+ expect(view(store)).toMatchObject({ entries: 0, held: 8 * MIB, charged: 8 * MIB })+ expect([...view(store).state.held.keys()].sort()).toEqual([...undone].sort())+ await expectRefused(store)+ await recoverAndWitness(store)+ })++ it.each(['partial', 'post-source-rename'] as const)(+ 'RC-FAILED-TRANSFER-%s: failed transfer trash keeps its former reservation charged',+ async stage => {+ const store = await storeWithBudget()+ const attempts = []+ for (const [index, caller] of ['a', 'b'].entries()) {+ const attempt = await startTransfer(store, roots[caller], caller, index, 4 * MIB)+ attempts.push({ ...attempt, caller })+ if (stage === 'post-source-rename') {+ const directory = downloadDirectory(roots[caller], attempt.transfer.id)+ boundaries.open.mockImplementation(async (...args: unknown[]) => {+ const handle = (await Reflect.apply(nativeFs.open, nativeFs, args)) as fs.FileHandle+ if (String(args[0]) === directory) vi.spyOn(handle, 'sync').mockRejectedValueOnce(eio())+ return handle+ })+ await expect(+ store.publish(attempt.transfer.id, caller, digestOf(attempt.bytes))+ ).rejects.toMatchObject({ code: 'storage_write_failed' })+ expect(syncFs.existsSync(path.join(directory, 'source'))).toBe(true)+ expect(syncFs.existsSync(path.join(directory, 'meta.json'))).toBe(true)+ restoreBoundary('open')+ }+ }+ expect(view(store).active).toBe(8 * MIB)+ const hits = failTrashRemoval()+ for (const attempt of attempts)+ await expect(store.fail(attempt.transfer.id, attempt.caller)).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ expect(hits).toHaveLength(2)+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ // Measured trees include meta.json, so the charge is at least the source bytes.+ expect(view(store).charged).toBeGreaterThanOrEqual(8 * MIB)+ await expectRefused(store)+ await recoverAndWitness(store)+ }+ )++ it('RC-NO-DOUBLE-CHARGE: repeated trash failures charge exactly once and admission at the remainder works', async () => {+ const store = await storeWithBudget()+ await expiredTrash(store)+ for (let i = 0; i < 4; i++) {+ expect((await store.cleanupExpired()).removeFailed).toBe(2)+ expect(view(store)).toMatchObject({ held: 8 * MIB, charged: 8 * MIB })+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ }+ await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ const fits = await startTransfer(store, roots.c, 'c', 4, 2 * MIB)+ expect(view(store).charged).toBe(10 * MIB)+ expect(storedBytes(hostRoot)).toBe(10 * MIB)+ restoreBoundary('rm')+ await store.fail(fits.transfer.id, 'c')+ await store.cleanupExpired()+ expect(view(store).charged).toBe(0)+ await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).resolves.toBeDefined()+ })++ it.each(['file', 'missing', 'parent-file'] as const)(+ 'RC-TRASH-ABSENCE-%s: definitive absence releases a held trash charge despite a failed listing',+ async replacement => {+ const store = await storeWithBudget()+ const hits = await expiredTrash(store)+ await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ const trash = [...view(store).state.held.keys()].find(p =>+ p.includes(`${path.sep}a${path.sep}`)+ )!+ expect(hits.length).toBeGreaterThanOrEqual(4)+ if (replacement === 'parent-file') {+ syncFs.rmSync(path.dirname(trash), { recursive: true, force: true })+ syncFs.writeFileSync(path.dirname(trash), '', { mode: 0o600 })+ await expect(nativeFs.lstat(trash)).rejects.toMatchObject({ code: 'ENOTDIR' })+ } else {+ syncFs.rmSync(trash, { recursive: true, force: true })+ if (replacement === 'file') syncFs.writeFileSync(trash, '', { mode: 0o600 })+ else await expect(nativeFs.lstat(trash)).rejects.toMatchObject({ code: 'ENOENT' })+ }+ const listed: string[] = []+ boundaries.readdir.mockImplementation((...args: unknown[]) => {+ if (String(args[0]) === path.join(roots.a, '.gfs-downloads')) {+ listed.push(String(args[0]))+ return Promise.reject(eio())+ }+ return Reflect.apply(nativeFs.readdir, nativeFs, args)+ })+ const fits = await startTransfer(store, roots.c, 'c', 3, 4 * MIB)+ expect(view(store)).toMatchObject({ held: 4 * MIB, active: 4 * MIB, charged: 8 * MIB })+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ if (replacement !== 'parent-file') expect(listed.length).toBeGreaterThan(0)+ restoreBoundary('rm')+ await store.fail(fits.transfer.id, 'c')+ }+ )++ it('RC-TRASH-UNKNOWN: failed listing and EIO probing retain trash charges until cleanup recovers', async () => {+ const store = await storeWithBudget()+ await expiredTrash(store)+ const trash = new Set(view(store).state.held.keys())+ const probed: string[] = []+ boundaries.readdir.mockImplementation((...args: unknown[]) => {+ if (String(args[0]).endsWith('.gfs-downloads')) return Promise.reject(eio())+ return Reflect.apply(nativeFs.readdir, nativeFs, args)+ })+ boundaries.lstat.mockImplementation((...args: unknown[]) => {+ if (trash.has(String(args[0]))) {+ probed.push(String(args[0]))+ return Promise.reject(eio())+ }+ return Reflect.apply(nativeFs.lstat, nativeFs, args)+ })+ await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(probed).toHaveLength(2)+ expect(view(store).held).toBe(8 * MIB)+ for (const name of ['readdir', 'lstat', 'rm'] as const) restoreBoundary(name)+ expect((await store.cleanupExpired()).removedIncomplete).toBe(2)+ expect(view(store).charged).toBe(0)+ const fresh = await completedCopy(store, roots.c, 'c', 3, 4 * MIB)+ expect((await store.readManagedFile(fresh.receipt.path, 'c')).equals(fresh.bytes)).toBe(true)+ })++ it('RC-SERIAL-SWEEPS: concurrent admission and cleanup queue behind the pre-check sweep and preserve live pins and active transfers', async () => {+ const store = await storeWithBudget()+ const expiry = new Date(Date.now() + 24 * HOUR).toISOString()+ const live = await completedCopy(store, roots.a, 'a', 0, 2 * MIB, {+ owner: 'live',+ expiresAt: expiry,+ })+ const active = await startTransfer(store, roots.a, 'a', 1, 2 * MIB, { expiresAt: expiry })+ const expiring = await completedCopy(store, roots.b, 'b', 2, 4 * MIB, { owner: 'expire' })+ vi.setSystemTime(Date.now() + 2 * HOUR)+ let enter!: () => void+ let release!: () => void+ const entered = new Promise<void>(resolve => {+ enter = resolve+ })+ const released = new Promise<void>(resolve => {+ release = resolve+ })+ releases.push(release)+ let armed = true+ boundaries.rm.mockImplementation(async (...args: unknown[]) => {+ if (armed && path.basename(String(args[0])).startsWith('.trash-')) {+ armed = false+ enter()+ await released+ }+ return Reflect.apply(nativeFs.rm, nativeFs, args)+ })+ boundaries.statfs.mockClear()+ const first = startTransfer(store, roots.c, 'c', 3, 4 * MIB)+ await entered+ let secondSettled = false+ let cleanupSettled = false+ const second = startTransfer(store, roots.d, 'd', 4, 4 * MIB).then(+ value => {+ secondSettled = true+ return value+ },+ error => {+ secondSettled = true+ return error as { code: string }+ }+ )+ const cleanup = store.cleanupExpired().then(value => {+ cleanupSettled = true+ return value+ })+ await new Promise<void>(resolve => setImmediate(resolve))+ expect(secondSettled).toBe(false)+ expect(cleanupSettled).toBe(false)+ expect(boundaries.statfs).not.toHaveBeenCalled()+ release()+ const admitted = await first+ await expect(second).resolves.toMatchObject({ code: 'download_busy' })+ await cleanup+ expect(secondSettled).toBe(true)+ expect(cleanupSettled).toBe(true)+ expect(syncFs.existsSync(downloadDirectory(roots.b, expiring.receipt.id))).toBe(false)+ expect(+ syncFs.readFileSync(path.join(roots.a, active.transfer.partialPath)).equals(active.bytes)+ ).toBe(true)+ expect((await store.readManagedFile(live.receipt.path, 'a')).equals(live.bytes)).toBe(true)+ await expect(store.readManagedFile(live.receipt.path, 'd')).rejects.toMatchObject({+ code: 'download_missing',+ message: 'GFS download store failed (download_missing)',+ })+ expect(view(store).charged).toBe(8 * MIB)+ await store.fail(active.transfer.id, 'a')+ const next = await startTransfer(store, roots.d, 'd', 4, 4 * MIB)+ expect(view(store).charged).toBe(10 * MIB)+ expect((await store.readManagedFile(live.receipt.path, 'a')).equals(live.bytes)).toBe(true)+ await store.fail(admitted.transfer.id, 'c')+ await store.fail(next.transfer.id, 'd')+ })++ it.each([false, true])(+ 'RC-REUSE-UNPINNED-%s: post-hash expiry is a miss with and without a new retention owner',+ async owner => {+ const store = await storeWithBudget()+ const copy = await completedCopy(store, roots.a, 'a', 0, 1024)+ await expect(store.reusableReceipt('a', sourceFor(0), 1024)).resolves.toMatchObject({+ id: copy.receipt.id,+ })+ vi.setSystemTime(Date.parse(copy.receipt.expiresAt) - 1)+ let enter!: () => void+ let release!: () => void+ const entered = new Promise<void>(resolve => {+ enter = resolve+ })+ const released = new Promise<void>(resolve => {+ release = resolve+ })+ releases.push(release)+ let armed = true+ boundaries.open.mockImplementation(async (...args: unknown[]) => {+ const handle = (await Reflect.apply(nativeFs.open, nativeFs, args)) as fs.FileHandle+ if (armed && String(args[0]).endsWith(`${path.sep}source`)) {+ armed = false+ const read = handle.read.bind(handle)+ vi.spyOn(handle, 'read').mockImplementationOnce(async (...readArgs: unknown[]) => {+ enter()+ await released+ return Reflect.apply(read, handle, readArgs)+ })+ }+ return handle+ })+ const reuse = store.reusableReceipt(+ 'a',+ sourceFor(0),+ 1024,+ owner ? { retentionOwnerId: 'new-owner' } : undefined+ )+ await entered+ vi.setSystemTime(Date.parse(copy.receipt.expiresAt))+ release()+ await expect(reuse).resolves.toBeUndefined()+ expect(view(store).state.pins.size).toBe(0)+ await store.cleanupExpired()+ const fresh = await completedCopy(store, roots.a, 'a', 0, 1024, { owner: 'new-owner' })+ expect((await store.readManagedFile(fresh.receipt.path, 'a')).equals(fresh.bytes)).toBe(true)+ }+ )++ it('RC-RESTART-TRASH: trash that survives startup cleanup remains charged after reopening the store', async () => {+ const original = await storeWithBudget()+ const hits = await expiredTrash(original)+ await original.close(0)+ const reopened = await storeWithBudget()+ expect(hits).toHaveLength(4)+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ expect(view(reopened).held).toBeGreaterThanOrEqual(8 * MIB)+ await expectRefused(reopened)+ await recoverAndWitness(reopened)+ })++ it('RC-FAIL-BEFORE-RENAME: a failed transfer still in its original incomplete directory retains its charge', async () => {+ const store = await storeWithBudget()+ const a = await startTransfer(store, roots.a, 'a', 1, 4 * MIB)+ const b = await startTransfer(store, roots.b, 'b', 2, 4 * MIB)+ const hits: string[] = []+ boundaries.rename.mockImplementation((...args: unknown[]) => {+ if (path.basename(String(args[0])).startsWith('input-')) {+ hits.push(String(args[0]))+ return Promise.reject(eio())+ }+ return Reflect.apply(nativeFs.rename, nativeFs, args)+ })+ await expect(store.fail(a.transfer.id, 'a')).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ await expect(store.fail(b.transfer.id, 'b')).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ expect(hits).toHaveLength(2)+ expect(storedBytes(hostRoot)).toBe(8 * MIB)+ expect(view(store).held).toBe(8 * MIB)+ await expectRefused(store)+ expect(hits).toHaveLength(4)+ await recoverAndWitness(store)+ })++ it.each([false, true])(+ 'RC-VOLUME-SECOND-CHECK-%s: the post-reclaim measurement retries invalid readings and retains the correct error',+ async persistent => {+ const store = await storeWithBudget()+ const valid = {+ ...(await nativeFs.statfs(hostRoot, { bigint: true })),+ bsize: 1n,+ blocks: TOTAL,+ bavail: TOTAL,+ }+ const invalid = { ...valid, bavail: -1n }+ boundaries.statfs.mockReset()+ boundaries.statfs+ .mockResolvedValueOnce(valid)+ .mockResolvedValueOnce(invalid)+ .mockResolvedValueOnce(persistent ? invalid : valid)+ const before = await quotaCount('host', 'free_space')+ const budgetBefore = await quotaCount('host', 'storage_bytes')+ const outcome = await startTransfer(store, roots.a, 'a', 1, 1024).then(+ admitted => ({ admitted }),+ error => error as { code: string; message: string }+ )+ expect(boundaries.statfs).toHaveBeenCalledTimes(3)+ expect(await quotaCount('host', 'storage_bytes')).toBe(budgetBefore)+ if (persistent) {+ expect(outcome).toMatchObject({+ code: 'volume_unmeasurable',+ message: 'GFS download store failed (volume_unmeasurable)',+ })+ expect(await quotaCount('host', 'free_space')).toBe(before + 1)+ expect(view(store).active).toBe(0)+ } else {+ expect('admitted' in outcome).toBe(true)+ expect(await quotaCount('host', 'free_space')).toBe(before)+ if ('admitted' in outcome) await store.fail(outcome.admitted.transfer.id, 'a')+ }+ boundaries.statfs.mockResolvedValue(valid)+ const fresh = await startTransfer(store, roots.a, 'a', 1, 1024)+ await store.fail(fresh.transfer.id, 'a')+ }+ )++ it.each([false, true])(+ 'RC-VOLUME-REUSE-%s: a new retention owner shares the invalid-statfs retry and category',+ async persistent => {+ const store = await storeWithBudget()+ const copy = await completedCopy(store, roots.a, 'a', 0, 1024)+ const valid = {+ ...(await nativeFs.statfs(hostRoot, { bigint: true })),+ bsize: 1n,+ blocks: TOTAL,+ bavail: TOTAL,+ }+ boundaries.statfs.mockReset()+ boundaries.statfs+ .mockResolvedValueOnce({ ...valid, bsize: 0n })+ .mockResolvedValueOnce(persistent ? { ...valid, bsize: 0n } : valid)+ const before = await quotaCount('host', 'free_space')+ const outcome = await store+ .reusableReceipt('a', sourceFor(0), 1024, { retentionOwnerId: 'new' })+ .then(+ receipt => ({ receipt }),+ error => error as { code: string; message: string }+ )+ expect(boundaries.statfs).toHaveBeenCalledTimes(2)+ if (persistent) {+ expect(outcome).toMatchObject({+ code: 'volume_unmeasurable',+ message: 'GFS download store failed (volume_unmeasurable)',+ })+ expect(await quotaCount('host', 'free_space')).toBe(before + 1)+ expect(view(store).state.pins.size).toBe(0)+ } else {+ expect(outcome).toMatchObject({ receipt: { id: copy.receipt.id } })+ expect(await quotaCount('host', 'free_space')).toBe(before)+ expect(view(store).state.pins.size).toBe(1)+ }+ boundaries.statfs.mockResolvedValue(valid)+ await expect(+ store.reusableReceipt('a', sourceFor(0), 1024, { retentionOwnerId: 'new' })+ ).resolves.toMatchObject({ id: copy.receipt.id })+ expect((await store.readManagedFile(copy.receipt.path, 'a')).equals(copy.bytes)).toBe(true)+ }+ )++ it.each([false, true])(+ 'RC-VOLUME-EIO-%s: a rejected statfs propagates without treating it as an invalid reading',+ async invalidFirst => {+ const store = await storeWithBudget()+ const valid = {+ ...(await nativeFs.statfs(hostRoot, { bigint: true })),+ bsize: 1n,+ blocks: TOTAL,+ bavail: TOTAL,+ }+ boundaries.statfs.mockReset()+ if (invalidFirst) boundaries.statfs.mockResolvedValueOnce({ ...valid, blocks: 0n })+ boundaries.statfs.mockRejectedValueOnce(eio())+ const before = await quotaCount('host', 'free_space')+ await expect(startTransfer(store, roots.a, 'a', 1, 1024)).rejects.toMatchObject({+ code: 'EIO',+ message: 'EIO: injected fault',+ })+ expect(boundaries.statfs).toHaveBeenCalledTimes(invalidFirst ? 2 : 1)+ expect(await quotaCount('host', 'free_space')).toBe(before)+ boundaries.statfs.mockResolvedValue(valid)+ const admitted = await startTransfer(store, roots.a, 'a', 1, 1024)+ await store.fail(admitted.transfer.id, 'a')+ }+ )+})diff --git a/mcp-host/src/internalTools/gfsDownloadStoreTrashAndExpiry.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreTrashAndExpiry.test.tsindex edb2ebb6d..b5e3b143f 100644--- a/mcp-host/src/internalTools/gfsDownloadStoreTrashAndExpiry.test.ts+++ b/mcp-host/src/internalTools/gfsDownloadStoreTrashAndExpiry.test.ts@@ -285,6 +285,92 @@ describe('GFS download store: a copy left under a trash name stays charged (R5-Fexpect(admitted.transfer.sizeBytes).toBe(4 * MIB)await store.fail(admitted.transfer.id, C)})++ it('OPUS-R6-RESTART: after a Host restart a trash directory whose removal keeps failing is sized and charged again', async () => {+ coherentVolume(1000n * BigInt(MIB), 1000n * BigInt(MIB))+ const store = await limitedStore({ MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT: '1' })+ await completedCopy(store, rootA, A, 0, 4 * MIB, { owner: 'task-a' })+ await completedCopy(store, rootB, B, 1, 4 * MIB, { owner: 'task-b' })+ vi.setSystemTime(Date.now() + 2 * HOUR_MS)+ const trashRemovals: string[] = []+ rmBoundary.mockImplementation((...args: unknown[]) => {+ if (path.basename(String(args[0])).startsWith('.trash-')) {+ trashRemovals.push(String(args[0]))+ return Promise.reject(Object.assign(new Error('EIO: injected'), { code: 'EIO' }))+ }+ return Reflect.apply(nativeFs.rm, nativeFs, args)+ })+ expect((await store.cleanupExpired()).removeFailed).toBe(2)+ await store.close(0)++ // A new process on the same root, the fault still present: it holds no+ // charge from the old one.+ const before = trashRemovals.length+ const restarted = await limitedStore({ MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT: '1' })+ // Witness: the startup sweep reached both trash directories and failed.+ expect(trashRemovals.length).toBe(before + 2)+ expect(trashIn(rootA)).toHaveLength(1)+ expect(trashIn(rootB)).toHaveLength(1)+ expect(storedSourceBytes(hostRoot)).toBe(8 * MIB)+ // 8 MiB of trash + 4 MiB requested > 10 MiB.+ await expect(startTransfer(restarted, rootC, C, 2, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(storedSourceBytes(hostRoot)).toBe(8 * MIB)++ // Fault cleared: the trash is removed and the same admission succeeds.+ rmBoundary.mockImplementation((...args: unknown[]) =>+ Reflect.apply(nativeFs.rm, nativeFs, args)+ )+ expect((await restarted.cleanupExpired()).removedIncomplete).toBe(2)+ expect(storedSourceBytes(hostRoot)).toBe(0)+ const admitted = await startTransfer(restarted, rootC, C, 2, 4 * MIB)+ expect(admitted.transfer.sizeBytes).toBe(4 * MIB)+ await restarted.fail(admitted.transfer.id, C)+ })++ it('DS-R6-FAIL-PARTIAL: a failed transfer whose trash cannot be removed stays charged with the bytes it wrote', async () => {+ coherentVolume(1000n * BigInt(MIB), 1000n * BigInt(MIB))+ const store = await limitedStore({ MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT: '1' })+ const first = await startTransfer(store, rootA, A, 0, 4 * MIB)+ const second = await startTransfer(store, rootB, B, 1, 4 * MIB)+ const trashRemovals: string[] = []+ rmBoundary.mockImplementation((...args: unknown[]) => {+ if (path.basename(String(args[0])).startsWith('.trash-')) {+ trashRemovals.push(String(args[0]))+ return Promise.reject(Object.assign(new Error('EIO: injected'), { code: 'EIO' }))+ }+ return Reflect.apply(nativeFs.rm, nativeFs, args)+ })++ // Neither transfer was published, so the store holds no indexed size for+ // either; both partials are left on disk under trash names.+ await expect(store.fail(first.transfer.id, A)).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ await expect(store.fail(second.transfer.id, B)).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ expect(trashRemovals).toHaveLength(2)+ expect(trashIn(rootA)).toHaveLength(1)+ expect(trashIn(rootB)).toHaveLength(1)+ expect(storedSourceBytes(hostRoot)).toBe(8 * MIB)++ // 8 MiB of partials + 4 MiB requested > 10 MiB.+ await expect(startTransfer(store, rootC, C, 2, 4 * MIB)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })++ // Fault cleared: the trash is removed and the same admission succeeds.+ rmBoundary.mockImplementation((...args: unknown[]) =>+ Reflect.apply(nativeFs.rm, nativeFs, args)+ )+ expect((await store.cleanupExpired()).removedIncomplete).toBe(2)+ expect(storedSourceBytes(hostRoot)).toBe(0)+ const admitted = await startTransfer(store, rootC, C, 2, 4 * MIB)+ expect(admitted.transfer.sizeBytes).toBe(4 * MIB)+ await store.fail(admitted.transfer.id, C)+ })})describe('GFS download store: reuse that crosses expiry while hashing (R5-F2)', () => {diff --git a/mcp-host/src/internalTools/gfsSpaceGuidance.ts b/mcp-host/src/internalTools/gfsSpaceGuidance.tsindex 4b9788042..14c2b9fbd 100644--- a/mcp-host/src/internalTools/gfsSpaceGuidance.ts+++ b/mcp-host/src/internalTools/gfsSpaceGuidance.ts@@ -13,9 +13,11 @@ export const GFS_DISK_FULL_GUIDANCE ='no longer need with a shell_exec command they approve. Never list, read or delete ' +"another user's directory or anything outside the workspace."-// The store evicts every copy no running task protects before it refuses, so-// deleting downloaded copies by hand cannot admit the download; only tasks-// finishing (or being cancelled) and copies expiring free that space.+// Deleting downloaded copies by hand cannot admit the download. The budget+// refusal comes only when evicting every copy no running task protects would+// still not make room, and the per-caller cap, checked before any eviction,+// counts only copies the caller's running tasks protect. Only tasks finishing+// (or being cancelled) and copies expiring free that space.export const GFS_CACHE_FULL_GUIDANCE ="The Host's cache of downloaded files is full, so the file was not downloaded. " +'Space frees as tasks finish and downloaded copies expire. Tell the user. ' +diff --git a/scripts/ci/pinned-vitest-cases/mcp-host.json b/scripts/ci/pinned-vitest-cases/mcp-host.jsonindex 4d841979b..4424e2c24 100644--- a/scripts/ci/pinned-vitest-cases/mcp-host.json+++ b/scripts/ci/pinned-vitest-cases/mcp-host.json@@ -1126,5 +1126,89 @@{"suite": "turnContext.test.ts","fullName": "prepared GFS receipts renders an unmeasurable volume without any guidance line"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-UNDO-FAIL: post-rename verification and undo failures retain the renamed bytes"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-FAILED-TRANSFER-partial: failed transfer trash keeps its former reservation charged"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-FAILED-TRANSFER-post-source-rename: failed transfer trash keeps its former reservation charged"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-NO-DOUBLE-CHARGE: repeated trash failures charge exactly once and admission at the remainder works"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-TRASH-ABSENCE-file: definitive absence releases a held trash charge despite a failed listing"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-TRASH-ABSENCE-missing: definitive absence releases a held trash charge despite a failed listing"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-TRASH-ABSENCE-parent-file: definitive absence releases a held trash charge despite a failed listing"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-TRASH-UNKNOWN: failed listing and EIO probing retain trash charges until cleanup recovers"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-SERIAL-SWEEPS: concurrent admission and cleanup queue behind the pre-check sweep and preserve live pins and active transfers"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-REUSE-UNPINNED-false: post-hash expiry is a miss with and without a new retention owner"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-REUSE-UNPINNED-true: post-hash expiry is a miss with and without a new retention owner"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-RESTART-TRASH: trash that survives startup cleanup remains charged after reopening the store"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-FAIL-BEFORE-RENAME: a failed transfer still in its original incomplete directory retains its charge"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-SECOND-CHECK-false: the post-reclaim measurement retries invalid readings and retains the correct error"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-SECOND-CHECK-true: the post-reclaim measurement retries invalid readings and retains the correct error"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-REUSE-false: a new retention owner shares the invalid-statfs retry and category"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-REUSE-true: a new retention owner shares the invalid-statfs retry and category"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-EIO-false: a rejected statfs propagates without treating it as an invalid reading"+ },+ {+ "suite": "gfsDownloadStoreRemovalCharge.test.ts",+ "fullName": "GFS download store removal charges RC-VOLUME-EIO-true: a rejected statfs propagates without treating it as an invalid reading"+ },+ {+ "suite": "gfsDownloadStoreTrashAndExpiry.test.ts",+ "fullName": "GFS download store: a copy left under a trash name stays charged (R5-F1) OPUS-R6-RESTART: after a Host restart a trash directory whose removal keeps failing is sized and charged again"+ },+ {+ "suite": "gfsDownloadStoreTrashAndExpiry.test.ts",+ "fullName": "GFS download store: a copy left under a trash name stays charged (R5-F1) DS-R6-FAIL-PARTIAL: a failed transfer whose trash cannot be removed stays charged with the bytes it wrote"}]