Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(mcp-host): add the filesystem-backed GFS download store
Alfredo Lopez09ce07d714ba
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.jsonindex 99097ec6f..611c9df81 100644--- a/mcp-host/package-lock.json+++ b/mcp-host/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-mcp-host",- "version": "0.1.521",+ "version": "0.1.522","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-mcp-host",- "version": "0.1.521",+ "version": "0.1.522","license": "SEE LICENSE IN LICENSE","dependencies": {"@anthropic-ai/sdk": "^0.32.0",diff --git a/mcp-host/package.json b/mcp-host/package.jsonindex bccf32be4..ad9532281 100644--- a/mcp-host/package.json+++ b/mcp-host/package.json@@ -1,6 +1,6 @@{"name": "clerum-mcp-host",- "version": "0.1.521",+ "version": "0.1.522","description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude","main": "dist/main.js","engines": {diff --git a/mcp-host/src/__tests__/fixtures/gfsStoreTestKit.ts b/mcp-host/src/__tests__/fixtures/gfsStoreTestKit.tsnew file mode 100644index 000000000..aa07269db--- /dev/null+++ b/mcp-host/src/__tests__/fixtures/gfsStoreTestKit.ts@@ -0,0 +1,203 @@+/**+ * Helpers for GFS download store tests that write the volume directly, the+ * way a shell command running with the Host UID can. They use the synchronous+ * `node:fs` API and `crypto.hash`, so a test that mocks `node:fs/promises` or+ * `createHash` to observe the store never counts the fixture's own calls.+ */+import { hash, randomUUID } from 'node:crypto'+import * as fs from 'node:fs'+import * as path from 'node:path'+import { register } from 'prom-client'+import type { GfsImageSource } from '../../visualInput/policy'++export function sourceFor(index: number, version = 1): GfsImageSource {+ const resourceId = index.toString(16).padStart(32, '0')+ return {+ kind: 'gfs',+ drive: 'main',+ resourceId,+ gfsUri: `gfs://main/${resourceId}`,+ name: `fixture-${index}.bin`,+ version,+ }+}++export function digestOf(bytes: Buffer): string {+ return hash('sha256', bytes, 'hex')+}++/** `<host>/users/<key>`, created private if missing. */+export function callerDirectory(host: string, key: string): string {+ const root = path.join(host, 'users', key)+ fs.mkdirSync(root, { recursive: true, mode: 0o700 })+ return root+}++export function downloadDirectory(root: string, id: string): string {+ return path.join(root, '.gfs-downloads', `input-${id}`)+}++export function exists(target: string): boolean {+ try {+ fs.lstatSync(target)+ return true+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false+ throw error+ }+}++/** A schema-1 meta.json body for `bytes`, valid for one hour from `nowMs`. */+export function metaFor(+ id: string,+ bytes: Buffer,+ nowMs: number,+ overrides: Record<string, unknown> = {}+): Record<string, unknown> {+ return {+ schemaVersion: 1,+ id,+ callerIdentity: 'planted-identity',+ source: sourceFor(900),+ sizeBytes: bytes.byteLength,+ sha256: digestOf(bytes),+ createdAt: new Date(nowMs).toISOString(),+ expiresAt: new Date(nowMs + 60 * 60_000).toISOString(),+ ...overrides,+ }+}++export interface PlantedDownload {+ id: string+ directory: string+ sourcePath: string+}++/**+ * Writes `<callerRoot>/.gfs-downloads/input-<id>/{meta.json, source}` without+ * the store. `meta: 'omit'` or `source: 'omit'` leaves that file out; a string+ * `meta` is written verbatim.+ */+export function plantDownload(+ callerRoot: string,+ options: {+ id?: string+ bytes: Buffer+ meta: Record<string, unknown> | string | 'omit'+ source?: 'write' | 'omit'+ directoryMode?: number+ sourceMode?: number+ }+): PlantedDownload {+ const id = options.id ?? randomUUID()+ const directory = downloadDirectory(callerRoot, id)+ fs.mkdirSync(path.dirname(directory), { recursive: true, mode: 0o700 })+ fs.chmodSync(path.dirname(directory), 0o700)+ fs.mkdirSync(directory, { mode: 0o700 })+ const sourcePath = path.join(directory, 'source')+ if (options.meta !== 'omit')+ fs.writeFileSync(+ path.join(directory, 'meta.json'),+ typeof options.meta === 'string' ? options.meta : JSON.stringify(options.meta),+ { mode: 0o600 }+ )+ if (options.source !== 'omit') {+ fs.writeFileSync(sourcePath, options.bytes, { mode: 0o600 })+ if (options.sourceMode !== undefined) fs.chmodSync(sourcePath, options.sourceMode)+ }+ if (options.directoryMode !== undefined) fs.chmodSync(directory, options.directoryMode)+ return { id, directory, sourcePath }+}++export async function metricValue(name: string, labels: Record<string, string>): Promise<number> {+ const metric = register.getSingleMetric(name)+ if (metric === undefined) throw new Error(`metric ${name} is not registered`)+ const { values } = await metric.get()+ return values+ .filter(sample => Object.entries(labels).every(([key, value]) => sample.labels[key] === value))+ .reduce((sum, sample) => sum + sample.value, 0)+}++export const expiryCount = (outcome: string) =>+ metricValue('clerum_gfs_download_expiry_total', { outcome })+export const quotaCount = (scope: string, reason: string) =>+ metricValue('clerum_gfs_download_quota_total', { scope, reason })++/** The store operations the shared transfer helpers need. */+export interface TransferringStore {+ createTransfer(input: {+ callerIdentity: string+ callerWorkspacePath: string+ source: GfsImageSource+ sizeBytes: number+ expiresAt: string+ retentionOwnerId?: string+ }): Promise<{+ id: string+ path: string+ partialPath: string+ sizeBytes: number+ expiresAt: string+ }>+ publish(+ id: string,+ callerIdentity: string,+ sha256: string+ ): Promise<{ id: string; path: string; sha256: string; expiresAt: string }>+}++/** Admits a transfer and writes its partial file; `fill` sets every byte. */+export async function startTransfer(+ store: TransferringStore,+ root: string,+ caller: string,+ index: number,+ sizeBytes: number,+ options: { version?: number; owner?: string; expiresAt?: string; fill?: number } = {}+) {+ const bytes = Buffer.alloc(sizeBytes, options.fill ?? (index % 251) + 1)+ const transfer = await store.createTransfer({+ callerIdentity: caller,+ callerWorkspacePath: root,+ source: sourceFor(index, options.version ?? 1),+ sizeBytes,+ expiresAt: options.expiresAt ?? new Date(Date.now() + 60 * 60_000).toISOString(),+ ...(options.owner === undefined ? {} : { retentionOwnerId: options.owner }),+ })+ fs.writeFileSync(path.join(root, transfer.partialPath), bytes)+ return { transfer, bytes }+}++export async function completedCopy(+ store: TransferringStore,+ root: string,+ caller: string,+ index: number,+ sizeBytes: number,+ options: { version?: number; owner?: string; expiresAt?: string; fill?: number } = {}+) {+ const { transfer, bytes } = await startTransfer(store, root, caller, index, sizeBytes, options)+ const receipt = await store.publish(transfer.id, caller, digestOf(bytes))+ return { receipt, bytes }+}++/**+ * Re-imports `modulePath` with environment limits applied, then restores the+ * environment. The imported store's logger is not the test's spied instance.+ */+export async function withEnvironment<T>(+ env: Record<string, string>,+ load: () => Promise<T>+): Promise<T> {+ const saved = { ...process.env }+ Object.assign(process.env, env)+ try {+ return await load()+ } finally {+ for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]+ for (const [key, value] of Object.entries(saved)) {+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ }+}diff --git a/mcp-host/src/internalTools/gfsDownloadMetrics.ts b/mcp-host/src/internalTools/gfsDownloadMetrics.tsindex ba12dce61..44c0008b2 100644--- a/mcp-host/src/internalTools/gfsDownloadMetrics.ts+++ b/mcp-host/src/internalTools/gfsDownloadMetrics.ts@@ -45,7 +45,12 @@ export type GfsDownloadQuotaReason =| 'active_downloads'| 'free_space'-export type GfsDownloadExpiryOutcome = 'expired_removed' | 'cleanup_failed' | 'sweep_failed'+export type GfsDownloadExpiryOutcome =+ | 'expired_removed'+ | 'incomplete_removed'+ | 'remove_failed'+ | 'cleanup_failed'+ | 'sweep_failed'const gfsDownloadAdmissionsTotal = getOrCreateCounter({name: 'clerum_gfs_download_admissions_total',diff --git a/mcp-host/src/internalTools/gfsDownloadStoreBoundaries.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreBoundaries.test.tsnew file mode 100644index 000000000..6bf9f791b--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreBoundaries.test.ts@@ -0,0 +1,881 @@+/**+ * Boundary values of the GFS download store: meta.json validation, quotas and+ * capacity, expiry instants, in-process concurrency, content integrity,+ * caller isolation and path swaps between a check and its use.+ * Every limit is exercised at the value that is accepted and at the first+ * value that is refused, on a real temporary directory.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as crypto from 'node:crypto'+import { randomUUID } from 'node:crypto'+import * as syncFs from 'node:fs'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import {+ callerDirectory,+ completedCopy,+ digestOf,+ downloadDirectory,+ exists,+ expiryCount,+ metaFor,+ plantDownload,+ quotaCount,+ sourceFor,+ startTransfer,+ withEnvironment,+} from '../__tests__/fixtures/gfsStoreTestKit'+import { logger } from '../logger'+import { GfsDownloadStoreError, GfsDownloadStoreFs } from './gfsDownloadStoreFs'+import { GFS_FILE_LIMITS, GFS_HOST_RETAINED_FILES } from './gfsFilePolicy'++const { statfsBoundary, renameBoundary, openBoundary, realpathBoundary } = vi.hoisted(() => ({+ statfsBoundary: vi.fn(),+ renameBoundary: vi.fn(),+ openBoundary: vi.fn(),+ realpathBoundary: vi.fn(),+}))+// Pass-through boundaries, overridden only to simulate statfs results and+// failures or to act between the two renames of a publication; open is only+// observed, to prove which files a lookup touched; realpath is where a path+// swap is injected right after the store's check returns.+vi.mock('node:fs/promises', async original => ({+ ...(await original<typeof fs>()),+ statfs: statfsBoundary,+ rename: renameBoundary,+ open: openBoundary,+ realpath: realpathBoundary,+}))++const A = 'caller-a'+const B = 'caller-b'+const HOUR = 60 * 60_000+const META_MAX_BYTES = 64 * 1024+const HOST_30_CALLER_15 = {+ MCP_HOST_GFS_MAX_FILE_BYTES: '10',+ MCP_HOST_GFS_DOWNLOAD_STORAGE_BYTES: '30',+ MCP_HOST_GFS_CALLER_DOWNLOAD_STORAGE_BYTES: '15',+}++let nativeFs: typeof fs+let hostRoot: string+let rootA: string+let rootB: string+let store: GfsDownloadStoreFs+const extraStores: GfsDownloadStoreFs[] = []+const outsideRoots: string[] = []++function receiptPathOf(id: string): string {+ return `.gfs-downloads/input-${id}/source`+}++function at(ms: number): string {+ return new Date(ms).toISOString()+}++/** Distinct timestamps so creation and last-use order are deterministic. */+function tick(): void {+ vi.setSystemTime(Date.now() + 1_000)+}++function plantComplete(root: string, identity: string, index: number, bytes: Buffer) {+ const id = randomUUID()+ return plantDownload(root, {+ id,+ bytes,+ meta: metaFor(id, bytes, Date.now(), { callerIdentity: identity, source: sourceFor(index) }),+ })+}++async function limitedStore(env: Record<string, string>): Promise<GfsDownloadStoreFs> {+ return withEnvironment(env, async () => {+ vi.resetModules()+ const { GfsDownloadStoreFs: Store } = await import('./gfsDownloadStoreFs')+ const opened = new Store(hostRoot) as unknown as GfsDownloadStoreFs+ extraStores.push(opened)+ await opened.initialize()+ return opened+ })+}++/** meta.json padded with an ignored field to exactly `bytes` bytes. */+function paddedMeta(id: string, bytes: Buffer, size: number): string {+ const base = JSON.stringify({ ...metaFor(id, bytes, Date.now()), padding: '' })+ const padded = JSON.stringify({+ ...metaFor(id, bytes, Date.now()),+ padding: 'x'.repeat(size - base.length),+ })+ expect(Buffer.byteLength(padded)).toBe(size)+ return padded+}++beforeEach(async () => {+ nativeFs = await vi.importActual<typeof fs>('node:fs/promises')+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))+ for (const boundary of [statfsBoundary, renameBoundary, openBoundary, realpathBoundary])+ boundary.mockReset()+ statfsBoundary.mockImplementation(nativeFs.statfs)+ renameBoundary.mockImplementation(nativeFs.rename)+ openBoundary.mockImplementation(nativeFs.open)+ realpathBoundary.mockImplementation(nativeFs.realpath)+ hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-boundaries-'))+ rootA = callerDirectory(hostRoot, A)+ rootB = callerDirectory(hostRoot, B)+ store = new GfsDownloadStoreFs(hostRoot)+ await store.initialize()+})++afterEach(async () => {+ vi.restoreAllMocks()+ vi.useRealTimers()+ for (const opened of extraStores.splice(0)) await opened.close(0)+ await store.close(0)+ await nativeFs.rm(hostRoot, { recursive: true, force: true })+ for (const outside of outsideRoots.splice(0))+ await nativeFs.rm(outside, { recursive: true, force: true })+})++describe('GFS download store boundaries: meta.json validation', () => {+ const retention = GFS_FILE_LIMITS.retentionMs+ const cases: Array<{+ name: string+ expected: 'removed' | 'retained'+ meta: (id: string, bytes: Buffer, now: number) => Record<string, unknown> | string+ }> = [+ {+ name: 'V1: malformed JSON is incomplete',+ expected: 'removed',+ meta: () => '{"schemaVersion":',+ },+ {+ name: 'V2: meta one byte over the size limit is incomplete',+ expected: 'removed',+ meta: (id, bytes) => paddedMeta(id, bytes, META_MAX_BYTES + 1),+ },+ {+ name: 'V3: meta exactly at the size limit is retained',+ expected: 'retained',+ meta: (id, bytes) => paddedMeta(id, bytes, META_MAX_BYTES),+ },+ {+ name: 'V4: wrong schemaVersion is incomplete',+ expected: 'removed',+ meta: (id, bytes, now) => metaFor(id, bytes, now, { schemaVersion: 2 }),+ },+ {+ name: 'V5: meta id that differs from its directory is incomplete',+ expected: 'removed',+ meta: (_id, bytes, now) => metaFor(randomUUID(), bytes, now),+ },+ {+ name: 'V6: expiresAt before createdAt is incomplete',+ expected: 'removed',+ meta: (id, bytes, now) => metaFor(id, bytes, now, { expiresAt: at(now - 1) }),+ },+ {+ name: 'V7: meta exactly at the retention bound is retained',+ expected: 'retained',+ meta: (id, bytes, now) => metaFor(id, bytes, now, { expiresAt: at(now + retention) }),+ },+ {+ name: 'V8: meta past the retention bound by 1 ms is incomplete',+ expected: 'removed',+ meta: (id, bytes, now) => metaFor(id, bytes, now, { expiresAt: at(now + retention + 1) }),+ },+ {+ name: 'V9: createdAt exactly one minute ahead is retained',+ expected: 'retained',+ meta: (id, bytes, now) =>+ metaFor(id, bytes, now, { createdAt: at(now + 60_000), expiresAt: at(now + HOUR) }),+ },+ {+ name: 'V10: createdAt more than one minute ahead is incomplete',+ expected: 'removed',+ meta: (id, bytes, now) =>+ metaFor(id, bytes, now, { createdAt: at(now + 60_001), expiresAt: at(now + HOUR) }),+ },+ {+ name: 'V12: sizeBytes different from the source size is incomplete',+ expected: 'removed',+ meta: (id, bytes, now) => metaFor(id, bytes, now, { sizeBytes: bytes.byteLength - 1 }),+ },+ ]++ it.each(cases)('$name', async ({ expected, meta }) => {+ const bytes = Buffer.alloc(16, 0x31)+ const id = randomUUID()+ const planted = plantDownload(rootA, { id, bytes, meta: meta(id, bytes, Date.now()) })+ const sibling = plantComplete(rootB, B, 300, Buffer.alloc(5, 0x32))++ const swept = await store.cleanupExpired()++ // Witness: the valid sibling plant is retained and counted.+ expect(exists(sibling.directory)).toBe(true)+ const inventory = await store.debugInventory()+ expect(inventory.byCaller.get(B)).toEqual({ files: 1, bytes: 5 })+ if (expected === 'removed') {+ expect(swept.removedIncomplete).toBe(1)+ expect(exists(planted.directory)).toBe(false)+ expect(inventory.files).toBe(1)+ } else {+ expect(swept.removedIncomplete).toBe(0)+ expect(exists(planted.directory)).toBe(true)+ expect(inventory.byCaller.get(A)).toEqual({ files: 1, bytes: 16 })+ }+ })++ it('V11: sizeBytes above maxFileBytes is incomplete', async () => {+ const tooLarge = Buffer.alloc(11, 0x33)+ const atLimit = Buffer.alloc(10, 0x34)+ const over = plantComplete(rootA, A, 301, tooLarge)+ const kept = plantComplete(rootB, B, 302, atLimit)+ const limited = await limitedStore(HOST_30_CALLER_15)++ expect(exists(over.directory)).toBe(false)+ // Witness: the copy exactly at maxFileBytes is retained.+ expect(exists(kept.directory)).toBe(true)+ expect((await limited.debugInventory()).files).toBe(1)+ })++ it('V13: an adopted entry with non-private modes is counted but never served', async () => {+ const bytes = Buffer.alloc(12, 0x35)+ const id = randomUUID()+ const planted = plantDownload(rootB, {+ id,+ bytes,+ meta: metaFor(id, bytes, Date.now(), { callerIdentity: B, source: sourceFor(303) }),+ directoryMode: 0o755,+ sourceMode: 0o644,+ })+ await store.cleanupExpired()++ expect((await store.debugInventory()).byCaller.get(B)).toEqual({ files: 1, bytes: 12 })+ await expect(store.reusableReceipt(B, sourceFor(303), 12)).resolves.toBeUndefined()+ await expect(store.readManagedFile(receiptPathOf(id), B)).rejects.toMatchObject({+ code: 'download_missing',+ })+ // Neither path touched it: an adopted copy is only counted and swept.+ expect(exists(planted.sourcePath)).toBe(true)+ expect(syncFs.statSync(planted.sourcePath).mode & 0o777).toBe(0o644)+ })++ it('V14: a published copy with non-private modes is not reused or read', async () => {+ const reused = await completedCopy(store, rootA, A, 304, 8)+ const reusedDirectory = downloadDirectory(rootA, reused.receipt.id)+ syncFs.chmodSync(path.join(reusedDirectory, 'source'), 0o644)+ await expect(store.reusableReceipt(A, sourceFor(304), 8)).resolves.toBeUndefined()+ // Witness: the reuse path opened the copy and removed it.+ expect(exists(reusedDirectory)).toBe(false)++ const read = await completedCopy(store, rootA, A, 305, 8)+ const readDirectory = downloadDirectory(rootA, read.receipt.id)+ syncFs.chmodSync(readDirectory, 0o755)+ await expect(store.readManagedFile(read.receipt.path, A)).rejects.toMatchObject({+ code: 'download_missing',+ })+ syncFs.chmodSync(readDirectory, 0o700)+ // Control: with the private mode back, the same read serves the bytes.+ await expect(store.readManagedFile(read.receipt.path, A)).resolves.toEqual(read.bytes)+ })++ it('V15: an admission past the retention window is refused', async () => {+ const now = Date.now()+ await expect(+ startTransfer(store, rootA, A, 306, 4, {+ expiresAt: at(now + GFS_FILE_LIMITS.retentionMs + 1),+ })+ ).rejects.toThrow(RangeError)+ const admitted = await startTransfer(store, rootA, A, 306, 4, {+ expiresAt: at(now + GFS_FILE_LIMITS.retentionMs),+ })+ expect(admitted.transfer.expiresAt).toBe(at(now + GFS_FILE_LIMITS.retentionMs))+ })+})++describe('GFS download store boundaries: quota and capacity', () => {+ it('Q1: caller bytes exactly at the budget are admitted and one byte more is refused', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ await completedCopy(limited, rootA, A, 310, 10, { owner: 'task-q1' })+ const denied = await quotaCount('caller', 'storage_bytes')++ const atBudget = await completedCopy(limited, rootA, A, 311, 5, { owner: 'task-q1' })+ expect(await quotaCount('caller', 'storage_bytes')).toBe(denied)+ await expect(limited.readManagedFile(atBudget.receipt.path, A)).resolves.toEqual(atBudget.bytes)++ await expect(startTransfer(limited, rootA, A, 312, 1)).rejects.toMatchObject({+ code: 'caller_quota_exceeded',+ })+ expect(await quotaCount('caller', 'storage_bytes')).toBe(denied + 1)+ })++ it('Q2: caller file count at the limit is refused and one below is admitted', async () => {+ const limited = await limitedStore({ MCP_HOST_GFS_CALLER_DOWNLOAD_MAX_FILES: '2' })+ await completedCopy(limited, rootA, A, 313, 4, { owner: 'task-q2' })+ const denied = await quotaCount('caller', 'retained_files')++ await completedCopy(limited, rootA, A, 314, 4, { owner: 'task-q2' })+ expect(await quotaCount('caller', 'retained_files')).toBe(denied)++ await expect(startTransfer(limited, rootA, A, 315, 4)).rejects.toMatchObject({+ code: 'caller_quota_exceeded',+ })+ expect(await quotaCount('caller', 'retained_files')).toBe(denied + 1)+ })++ it('Q3: Host bytes exactly at the budget are admitted and one byte more is refused', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ const rootC = callerDirectory(hostRoot, 'caller-c')+ await completedCopy(limited, rootB, B, 316, 10, { owner: 'task-b' })+ await completedCopy(limited, rootC, 'caller-c', 317, 10, { owner: 'task-c' })+ const denied = await quotaCount('host', 'storage_bytes')++ await completedCopy(limited, rootA, A, 318, 10, { owner: 'task-a' })+ expect(await quotaCount('host', 'storage_bytes')).toBe(denied)+ expect((await limited.debugInventory()).bytes).toBe(30)++ const rootD = callerDirectory(hostRoot, 'caller-d')+ await expect(startTransfer(limited, rootD, 'caller-d', 319, 1)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(await quotaCount('host', 'storage_bytes')).toBe(denied + 1)+ })++ it('Q4: Host file count at the limit is refused and one below is admitted', async () => {+ const perCaller = GFS_FILE_LIMITS.callerRetainedFiles+ let index = 400+ for (let n = 0; n < GFS_HOST_RETAINED_FILES - 1; n += 1) {+ const caller = `caller-${Math.floor(n / perCaller)}`+ await completedCopy(store, callerDirectory(hostRoot, caller), caller, index, 1, {+ owner: `task-${caller}`,+ })+ index += 1+ }+ expect((await store.debugInventory()).files).toBe(GFS_HOST_RETAINED_FILES - 1)+ const denied = await quotaCount('host', 'retained_files')+ const lastCaller = `caller-${Math.floor((GFS_HOST_RETAINED_FILES - 1) / perCaller)}`++ await completedCopy(store, callerDirectory(hostRoot, lastCaller), lastCaller, index, 1, {+ owner: `task-${lastCaller}`,+ })+ expect(await quotaCount('host', 'retained_files')).toBe(denied)+ expect((await store.debugInventory()).files).toBe(GFS_HOST_RETAINED_FILES)++ const fresh = 'caller-fresh'+ await expect(+ startTransfer(store, callerDirectory(hostRoot, fresh), fresh, index + 1, 1)+ ).rejects.toMatchObject({ code: 'host_quota_exceeded' })+ expect(await quotaCount('host', 'retained_files')).toBe(denied + 1)+ })++ it('Q5: free space exactly at the threshold is admitted and one byte less is refused', async () => {+ const observed = await nativeFs.statfs(hostRoot, { bigint: true })+ const size = 100+ const threshold = BigInt(size) + 16n * 1024n * 1024n+ statfsBoundary.mockResolvedValue({ ...observed, bsize: 1n, bavail: threshold - 1n })+ const denied = await quotaCount('host', 'free_space')++ await expect(startTransfer(store, rootA, A, 320, size)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(await quotaCount('host', 'free_space')).toBe(denied + 1)++ statfsBoundary.mockResolvedValue({ ...observed, bsize: 1n, bavail: threshold })+ await expect(startTransfer(store, rootA, A, 320, size)).resolves.toBeDefined()+ expect(await quotaCount('host', 'free_space')).toBe(denied + 1)+ })++ it('Q6: a statfs failure refuses that admission loudly and the next one succeeds', async () => {+ statfsBoundary.mockRejectedValueOnce(Object.assign(new Error('injected EIO'), { code: 'EIO' }))++ await expect(startTransfer(store, rootA, A, 321, 4)).rejects.toMatchObject({ code: 'EIO' })+ expect(statfsBoundary).toHaveBeenCalledTimes(1)++ // The caller's single concurrency slot was not kept by the failed admission.+ const next = await completedCopy(store, rootA, A, 321, 4)+ await expect(store.readManagedFile(next.receipt.path, A)).resolves.toEqual(next.bytes)+ })++ it('Q7: a zero-byte file publishes, reads, reuses and has an empty prefix', async () => {+ const { receipt } = await completedCopy(store, rootA, A, 322, 0)++ expect(receipt.sha256).toBe(digestOf(Buffer.alloc(0)))+ await expect(store.readManagedFile(receipt.path, A)).resolves.toEqual(Buffer.alloc(0))+ await expect(store.readManagedFilePrefix(receipt.path, A)).resolves.toEqual(Buffer.alloc(0))+ await expect(store.reusableReceipt(A, sourceFor(322), 0)).resolves.toMatchObject({+ id: receipt.id,+ })+ })++ it('Q9: Host eviction takes adopted copies before any published copy', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ const published = await completedCopy(limited, rootB, B, 323, 10)+ tick()+ const rootC = callerDirectory(hostRoot, 'caller-c')+ const adopted = plantComplete(rootC, 'caller-c', 324, Buffer.alloc(10, 0x36))+ tick()+ await completedCopy(limited, rootA, A, 325, 10)+ tick()++ // caller-a stays within 15; the Host would reach 35 of 30.+ await startTransfer(limited, rootA, A, 326, 5)++ expect(exists(adopted.directory)).toBe(false)+ // Witness: the older published copy of another caller stays.+ expect(exists(downloadDirectory(rootB, published.receipt.id))).toBe(true)+ })++ it('Q10: caller eviction takes adopted copies in its own directory before its published copies', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ const published = await completedCopy(limited, rootA, A, 327, 5)+ tick()+ const adopted = plantComplete(rootA, A, 328, Buffer.alloc(5, 0x37))+ const foreign = plantComplete(rootB, B, 329, Buffer.alloc(5, 0x38))+ tick()++ // 5 + 5 + 10 against 15: one copy of caller-a's directory goes.+ await startTransfer(limited, rootA, A, 330, 10)++ expect(exists(adopted.directory)).toBe(false)+ expect(exists(downloadDirectory(rootA, published.receipt.id))).toBe(true)+ // Witness: a caller denial never reaches another directory's adopted copy.+ expect(exists(foreign.directory)).toBe(true)+ })++ it('Q11: published copies are evicted least recently used first', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ const older = await completedCopy(limited, rootA, A, 331, 5)+ tick()+ const newer = await completedCopy(limited, rootA, A, 332, 5)+ tick()+ await expect(limited.reusableReceipt(A, sourceFor(331), 5)).resolves.toMatchObject({+ id: older.receipt.id,+ })+ tick()++ await startTransfer(limited, rootA, A, 333, 10)++ expect(exists(downloadDirectory(rootA, newer.receipt.id))).toBe(false)+ expect(exists(downloadDirectory(rootA, older.receipt.id))).toBe(true)+ })++ it('Q12: when every copy is pinned admission fails with the quota code and succeeds after release', async () => {+ const limited = await limitedStore(HOST_30_CALLER_15)+ const pinned = await completedCopy(limited, rootA, A, 334, 10, { owner: 'task-q12' })+ const directory = downloadDirectory(rootA, pinned.receipt.id)+ const denied = await quotaCount('caller', 'storage_bytes')++ await expect(startTransfer(limited, rootA, A, 335, 10)).rejects.toMatchObject({+ code: 'caller_quota_exceeded',+ })+ expect(await quotaCount('caller', 'storage_bytes')).toBe(denied + 1)+ expect(exists(directory)).toBe(true)++ await limited.releaseReceiptOwner('task-q12', A)+ await expect(startTransfer(limited, rootA, A, 335, 10)).resolves.toBeDefined()+ expect(exists(directory)).toBe(false)+ expect(await quotaCount('caller', 'storage_bytes')).toBe(denied + 1)+ })+})++describe('GFS download store boundaries: expiry', () => {+ async function expiringCopy(index: number) {+ const expiresAtMs = Date.now() + HOUR+ const copy = await completedCopy(store, rootA, A, index, 8, { expiresAt: at(expiresAtMs) })+ return { ...copy, expiresAtMs, directory: downloadDirectory(rootA, copy.receipt.id) }+ }++ it('E1: reuse matches until one millisecond before expiry', async () => {+ const copy = await expiringCopy(340)+ vi.setSystemTime(copy.expiresAtMs - 1)+ await expect(store.reusableReceipt(A, sourceFor(340), 8)).resolves.toMatchObject({+ id: copy.receipt.id,+ })+ vi.setSystemTime(copy.expiresAtMs)+ await expect(store.reusableReceipt(A, sourceFor(340), 8)).resolves.toBeUndefined()+ vi.setSystemTime(copy.expiresAtMs + 1)+ await expect(store.reusableReceipt(A, sourceFor(340), 8)).resolves.toBeUndefined()+ })++ it('E2: a managed read succeeds until one millisecond before expiry', async () => {+ const copy = await expiringCopy(341)+ vi.setSystemTime(copy.expiresAtMs - 1)+ await expect(store.readManagedFile(copy.receipt.path, A)).resolves.toEqual(copy.bytes)+ for (const instant of [copy.expiresAtMs, copy.expiresAtMs + 1]) {+ vi.setSystemTime(instant)+ await expect(store.readManagedFile(copy.receipt.path, A)).rejects.toMatchObject({+ code: 'download_expired',+ })+ }+ })++ it('E3: the sweep removes a copy exactly at its expiry', async () => {+ const copy = await expiringCopy(342)+ await expect(store.cleanupExpired(copy.expiresAtMs - 1)).resolves.toMatchObject({+ removedExpired: 0,+ })+ expect(exists(copy.directory)).toBe(true)+ await expect(store.cleanupExpired(copy.expiresAtMs)).resolves.toMatchObject({+ removedExpired: 1,+ })+ expect(exists(copy.directory)).toBe(false)+ })+})++describe('GFS download store boundaries: in-process concurrency', () => {+ it('C1: two concurrent downloads of one source by one caller both publish and reuse returns one of them', async () => {+ const limited = await limitedStore({ MCP_HOST_GFS_CALLER_DOWNLOAD_CONCURRENCY: '2' })+ const [first, second] = await Promise.all([+ completedCopy(limited, rootA, A, 350, 6),+ completedCopy(limited, rootA, A, 350, 6),+ ])++ expect(first.receipt.id).not.toBe(second.receipt.id)+ expect((await limited.debugInventory()).byCaller.get(A)).toEqual({ files: 2, bytes: 12 })+ const reused = await limited.reusableReceipt(A, sourceFor(350), 6)+ expect([first.receipt.id, second.receipt.id]).toContain(reused?.id)+ })++ it('C2: two callers downloading one source each reuse their own copy', async () => {+ const [own, other] = await Promise.all([+ completedCopy(store, rootA, A, 351, 6),+ completedCopy(store, rootB, B, 351, 6),+ ])++ await expect(store.reusableReceipt(A, sourceFor(351), 6)).resolves.toMatchObject({+ id: own.receipt.id,+ })+ await expect(store.reusableReceipt(B, sourceFor(351), 6)).resolves.toMatchObject({+ id: other.receipt.id,+ })+ })++ it('C3: a sweep during an active transfer leaves its partial file alone', async () => {+ const { transfer, bytes } = await startTransfer(store, rootA, A, 352, 12)+ const stale = plantDownload(rootB, { bytes: Buffer.alloc(3), meta: 'omit' })++ // Witness: the same sweep removed the unrelated incomplete directory.+ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(exists(stale.directory)).toBe(false)+ expect(exists(path.join(rootA, transfer.partialPath))).toBe(true)+ const receipt = await store.publish(transfer.id, A, digestOf(bytes))+ await expect(store.readManagedFile(receipt.path, A)).resolves.toEqual(bytes)+ })++ it('C4: a sweep requested during publication runs after it and keeps the copy', async () => {+ const { transfer, bytes } = await startTransfer(store, rootA, A, 353, 12)+ let sweep: ReturnType<GfsDownloadStoreFs['cleanupExpired']> | undefined+ renameBoundary.mockImplementation(async (from: string, to: string) => {+ await nativeFs.rename(from, to)+ // Between the two renames: meta.json is visible, source is not yet.+ if (path.basename(to) === 'meta.json') sweep = store.cleanupExpired()+ })++ const receipt = await store.publish(transfer.id, A, digestOf(bytes))++ expect(sweep).toBeDefined()+ await expect(sweep).resolves.toMatchObject({ removedIncomplete: 0, removedExpired: 0 })+ await expect(store.readManagedFile(receipt.path, A)).resolves.toEqual(bytes)+ })++ it('C5: close requested during publication waits for it', async () => {+ const { transfer, bytes } = await startTransfer(store, rootA, A, 354, 12)+ let closing: Promise<void> | undefined+ renameBoundary.mockImplementation(async (from: string, to: string) => {+ await nativeFs.rename(from, to)+ if (path.basename(to) === 'meta.json') closing = store.close(5_000)+ })++ const receipt = await store.publish(transfer.id, A, digestOf(bytes))+ expect(closing).toBeDefined()+ await closing++ expect(store.isAvailable()).toBe(false)+ expect(syncFs.readFileSync(path.join(rootA, receipt.path))).toEqual(bytes)+ })++ it('C7: fail() after a partial write removes the directory and releases the reservation', async () => {+ const { transfer } = await startTransfer(store, rootA, A, 355, 12)+ const directory = downloadDirectory(rootA, transfer.id)+ syncFs.truncateSync(path.join(rootA, transfer.partialPath), 5)++ await store.fail(transfer.id, A)++ expect(exists(directory)).toBe(false)+ const next = await completedCopy(store, rootA, A, 355, 12)+ await expect(store.readManagedFile(next.receipt.path, A)).resolves.toEqual(next.bytes)+ })++ it('C8: fail() after publish is refused and the copy stays readable', async () => {+ const { receipt, bytes } = await completedCopy(store, rootA, A, 356, 12)++ await expect(store.fail(receipt.id, A)).rejects.toMatchObject({ code: 'download_busy' })++ await expect(store.readManagedFile(receipt.path, A)).resolves.toEqual(bytes)+ })+})++describe('GFS download store boundaries: integrity', () => {+ const damage = {+ truncated: (source: string) => syncFs.truncateSync(source, 7),+ 'replaced by a directory': (source: string) => {+ syncFs.rmSync(source)+ syncFs.mkdirSync(source, { mode: 0o700 })+ },+ }++ it.each([+ { name: 'I2: truncated source on reuse', kind: 'truncated' as const },+ {+ name: 'I4: source replaced by a directory on reuse',+ kind: 'replaced by a directory' as const,+ },+ ])('$name', async ({ kind }) => {+ const { receipt } = await completedCopy(store, rootA, A, 360, 8)+ const directory = downloadDirectory(rootA, receipt.id)+ damage[kind](path.join(directory, 'source'))++ await expect(store.reusableReceipt(A, sourceFor(360), 8)).resolves.toBeUndefined()++ expect(exists(directory)).toBe(false)+ const fresh = await completedCopy(store, rootA, A, 360, 8)+ expect(fresh.receipt.id).not.toBe(receipt.id)+ })++ it.each([+ { name: 'I3: truncated source on managed read', kind: 'truncated' as const },+ {+ name: 'I5: source replaced by a directory on managed read',+ kind: 'replaced by a directory' as const,+ },+ ])('$name', async ({ kind }) => {+ const { receipt } = await completedCopy(store, rootA, A, 361, 8)+ const directory = downloadDirectory(rootA, receipt.id)+ damage[kind](path.join(directory, 'source'))++ await expect(store.readManagedFile(receipt.path, A)).rejects.toBeInstanceOf(+ GfsDownloadStoreError+ )+ await expect(store.readManagedFile(receipt.path, A)).rejects.toMatchObject({+ code: 'download_missing',+ })+ await expect(store.readManagedFilePrefix(receipt.path, A)).rejects.toMatchObject({+ code: 'download_missing',+ })++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })+ expect(exists(directory)).toBe(false)+ })+})++/** Everything a caller can observe about a rejection, minus the stack. */+async function rejectionOf(operation: Promise<unknown>): Promise<Record<string, unknown>> {+ const error = await operation.then(+ () => {+ throw new Error('expected a rejection')+ },+ (rejection: unknown) => rejection+ )+ if (!(error instanceof Error)) throw new Error('expected an Error rejection')+ const fields = Object.getOwnPropertyNames(error).filter(name => name !== 'stack')+ return {+ constructor: error.constructor,+ fields: Object.fromEntries(+ fields.map(name => [name, (error as unknown as Record<string, unknown>)[name]])+ ),+ name: error.name,+ message: error.message,+ }+}++function opensUnder(id: string): number {+ return openBoundary.mock.calls.filter(([target]) => String(target).includes(id)).length+}++describe('GFS download store boundaries: caller isolation', () => {+ const reads = {+ readManagedFile: (target: GfsDownloadStoreFs, receiptPath: string, caller: string) =>+ target.readManagedFile(receiptPath, caller),+ readManagedFilePrefix: (target: GfsDownloadStoreFs, receiptPath: string, caller: string) =>+ target.readManagedFilePrefix(receiptPath, caller, 4),+ } as const++ it.each(Object.keys(reads) as Array<keyof typeof reads>)(+ "U1: %s answers another caller's copy exactly like a nonexistent one, without opening it",+ async api => {+ const { receipt, bytes } = await completedCopy(store, rootA, A, 70, 8)+ openBoundary.mockClear()+ const foreign = await rejectionOf(reads[api](store, receipt.path, B))+ const missing = await rejectionOf(reads[api](store, receiptPathOf(randomUUID()), B))++ expect(foreign).toStrictEqual(missing)+ expect(foreign).toMatchObject({ fields: { code: 'download_missing' } })+ expect(String(foreign.message)).not.toContain(receipt.id)+ expect(opensUnder(receipt.id)).toBe(0)+ // Witness: the owner reads the same path, and that read opens the copy.+ const owned = await reads[api](store, receipt.path, A)+ expect(owned).toEqual(api === 'readManagedFile' ? bytes : bytes.subarray(0, 4))+ expect(opensUnder(receipt.id)).toBeGreaterThan(0)+ }+ )++ it('U2: an expired copy of another caller is still answered as nonexistent, not as expired', async () => {+ const { receipt } = await completedCopy(store, rootA, A, 71, 8)+ vi.setSystemTime(Date.parse(receipt.expiresAt))++ const foreign = await rejectionOf(store.readManagedFile(receipt.path, B))+ const missing = await rejectionOf(store.readManagedFile(receiptPathOf(randomUUID()), B))++ expect(foreign).toStrictEqual(missing)+ // Witness: the owner sees the expiry.+ await expect(store.readManagedFile(receipt.path, A)).rejects.toMatchObject({+ code: 'download_expired',+ })+ })++ it("U3: publish and fail answer another caller's transfer exactly like an unknown id", async () => {+ const { transfer, bytes } = await startTransfer(store, rootA, A, 72, 8)+ const sha256 = digestOf(bytes)++ expect(await rejectionOf(store.publish(transfer.id, B, sha256))).toStrictEqual(+ await rejectionOf(store.publish(randomUUID(), B, sha256))+ )+ expect(await rejectionOf(store.fail(transfer.id, B))).toStrictEqual(+ await rejectionOf(store.fail(randomUUID(), B))+ )+ // Witness: the transfer was live and still publishes for its owner.+ const receipt = await store.publish(transfer.id, A, sha256)+ expect(receipt.id).toBe(transfer.id)+ expect(await rejectionOf(store.publish(transfer.id, B, sha256))).toStrictEqual(+ await rejectionOf(store.publish(randomUUID(), B, sha256))+ )+ expect(await rejectionOf(store.fail(transfer.id, B))).toStrictEqual(+ await rejectionOf(store.fail(randomUUID(), B))+ )+ })++ it('U4: an owner id another caller pins is neither refused nor released across callers', async () => {+ const pinnedA = await completedCopy(store, rootA, A, 73, 8, { owner: 'task-73' })+ const pinnedB = await completedCopy(store, rootB, B, 74, 8, { owner: 'task-73' })+ const reused = await store.reusableReceipt(B, sourceFor(74), 8, { retentionOwnerId: 'task-73' })+ expect(reused?.id).toBe(pinnedB.receipt.id)++ await expect(store.releaseReceiptOwner('task-73', B)).resolves.toBeUndefined()+ await expect(store.releaseReceiptOwner('task-73', B)).resolves.toBeUndefined()+ const later = Date.parse(pinnedA.receipt.expiresAt) + HOUR++ // Witness: B's release freed B's copy only; A's pin still holds A's.+ await expect(store.cleanupExpired(later)).resolves.toMatchObject({ removedExpired: 1 })+ expect(exists(downloadDirectory(rootB, pinnedB.receipt.id))).toBe(false)+ expect(exists(downloadDirectory(rootA, pinnedA.receipt.id))).toBe(true)+ })+})++/**+ * Replaces `users/<B>` by a symlink to a directory outside the Host root the+ * first time the store's realpath of `suffix` returns: the check has passed,+ * the use has not happened yet.+ */+function swapAfterCheck(suffix: string, outside: string): { swapped: () => boolean } {+ let swapped = false+ realpathBoundary.mockImplementation(async (target: string) => {+ const real = await nativeFs.realpath(target)+ if (!swapped && String(target).endsWith(suffix)) {+ swapped = true+ syncFs.renameSync(rootB, `${rootB}-moved`)+ syncFs.symlinkSync(outside, rootB)+ }+ return real+ })+ return { swapped: () => swapped }+}++function outsideDirectory(): string {+ const outside = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-swap-'))+ outsideRoots.push(outside)+ return outside+}++describe('GFS download store boundaries: path swapped between check and use', () => {+ it('K1: a removal whose parent is swapped after the check renames back and touches nothing outside', async () => {+ const { receipt, bytes } = await completedCopy(store, rootB, B, 80, 8)+ const outside = outsideDirectory()+ const decoy = plantDownload(outside, {+ id: receipt.id,+ bytes,+ meta: metaFor(receipt.id, bytes, Date.now(), { callerIdentity: B, source: sourceFor(80) }),+ })+ const failed = await expiryCount('remove_failed')+ const warn = vi.spyOn(logger, 'warn')+ const swap = swapAfterCheck(`/users/${B}/.gfs-downloads`, outside)+ renameBoundary.mockClear()++ await expect(store.cleanupExpired(Date.parse(receipt.expiresAt))).resolves.toMatchObject({+ removedExpired: 0,+ removeFailed: 1,+ })++ // Witness: the swap happened after the check and the trash rename was reached.+ expect(swap.swapped()).toBe(true)+ const renames = renameBoundary.mock.calls.map(([, to]) => path.basename(String(to)))+ expect(renames.some(name => name.startsWith('.trash-'))).toBe(true)+ expect(await expiryCount('remove_failed')).toBe(failed + 1)+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ component: 'GfsDownloadStore', code: 'workspace_unavailable' }),+ expect.stringContaining('could not remove')+ )+ expect(syncFs.readFileSync(decoy.sourcePath)).toEqual(bytes)+ expect(syncFs.readdirSync(path.dirname(decoy.directory))).toEqual([`input-${receipt.id}`])+ })++ it('K2: a publication whose caller directory is swapped after the check writes nothing outside', async () => {+ const { transfer, bytes } = await startTransfer(store, rootB, B, 81, 8)+ const outside = outsideDirectory()+ const decoy = downloadDirectory(outside, transfer.id)+ syncFs.mkdirSync(decoy, { recursive: true, mode: 0o700 })+ syncFs.chmodSync(path.dirname(decoy), 0o700)+ syncFs.writeFileSync(path.join(decoy, 'source.partial'), bytes, { mode: 0o600 })+ const swap = swapAfterCheck(`/users/${B}`, outside)+ openBoundary.mockClear()++ await expect(store.publish(transfer.id, B, digestOf(bytes))).rejects.toMatchObject({+ code: 'workspace_unavailable',+ })++ // Witness: the swap happened and the store opened the decoy's partial.+ expect(swap.swapped()).toBe(true)+ const opened = openBoundary.mock.calls.map(([target]) => String(target))+ expect(opened.some(target => target.endsWith(`input-${transfer.id}/source.partial`))).toBe(true)+ expect(syncFs.readdirSync(decoy)).toEqual(['source.partial'])+ expect(syncFs.readFileSync(path.join(decoy, 'source.partial'))).toEqual(bytes)+ })++ it('K3: a read whose caller directory is swapped after the check serves nothing from outside', async () => {+ const { receipt, bytes } = await completedCopy(store, rootB, B, 82, 8)+ // Witness: the owner's read works before the swap.+ await expect(store.readManagedFilePrefix(receipt.path, B, 4)).resolves.toEqual(+ bytes.subarray(0, 4)+ )+ const outside = outsideDirectory()+ const foreignBytes = Buffer.alloc(8, 0x5c)+ const decoy = plantDownload(outside, {+ id: receipt.id,+ bytes: foreignBytes,+ meta: metaFor(receipt.id, foreignBytes, Date.now(), { callerIdentity: B }),+ })+ const swap = swapAfterCheck(`/users/${B}`, outside)+ openBoundary.mockClear()++ await expect(store.readManagedFilePrefix(receipt.path, B, 4)).rejects.toMatchObject({+ code: 'download_missing',+ })++ // Witness: the swap happened and the store opened the decoy's source.+ expect(swap.swapped()).toBe(true)+ const opened = openBoundary.mock.calls.map(([target]) => String(target))+ expect(opened.some(target => target.endsWith(`input-${receipt.id}/source`))).toBe(true)+ expect(syncFs.readFileSync(decoy.sourcePath)).toEqual(foreignBytes)+ })+})diff --git a/mcp-host/src/internalTools/gfsDownloadStoreFs.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreFs.test.tsnew file mode 100644index 000000000..5a9fa3e80--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreFs.test.ts@@ -0,0 +1,811 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as crypto from 'node:crypto'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import { register } from 'prom-client'+import { logger } from '../logger'+import type { GfsImageSource } from '../visualInput/policy'+import { GfsDownloadStoreError, GfsDownloadStoreFs } from './gfsDownloadStoreFs'++const { statfsBoundary, renameBoundary, rmBoundary, lstatBoundary, hashBoundary } = vi.hoisted(+ () => ({+ statfsBoundary: vi.fn(),+ renameBoundary: vi.fn(),+ rmBoundary: vi.fn(),+ lstatBoundary: vi.fn(),+ hashBoundary: vi.fn(),+ })+)+// Pass-through boundaries: every test starts with the real implementation and+// overrides one call only where it simulates a filesystem failure or observes+// an order. Test setup and assertions use the actual modules.+vi.mock('node:fs/promises', async original => ({+ ...(await original<typeof fs>()),+ statfs: statfsBoundary,+ rename: renameBoundary,+ rm: rmBoundary,+ lstat: lstatBoundary,+}))+vi.mock('node:crypto', async original => ({+ ...(await original<typeof crypto>()),+ createHash: hashBoundary,+}))++let nativeFs: typeof fs+let nativeCrypto: typeof crypto+let hostRoot: string+let callerRoot: string+let store: GfsDownloadStoreFs+const extraStores: GfsDownloadStoreFs[] = []++const CALLER = 'caller-a'++function sourceFor(index: number, version = 1): GfsImageSource {+ const resourceId = index.toString(16).padStart(32, '0')+ return {+ kind: 'gfs',+ drive: 'main',+ resourceId,+ gfsUri: `gfs://main/${resourceId}`,+ name: `fixture-${index}.bin`,+ version,+ }+}++function future(ms = 60 * 60_000): string {+ return new Date(Date.now() + ms).toISOString()+}++function digest(bytes: Buffer): string {+ return nativeCrypto.createHash('sha256').update(bytes).digest('hex')+}++async function callerDirectory(host: string, caller: string): Promise<string> {+ const root = path.join(host, 'users', caller)+ await nativeFs.mkdir(root, { recursive: true, mode: 0o700 })+ return root+}++function downloadDirectory(root: string, id: string): string {+ return path.join(root, '.gfs-downloads', `input-${id}`)+}++async function exists(target: string): Promise<boolean> {+ try {+ await nativeFs.lstat(target)+ return true+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false+ throw error+ }+}++async function startTransfer(+ target: GfsDownloadStoreFs,+ root: string,+ caller: string,+ index: number,+ sizeBytes: number,+ options: { version?: number; owner?: string } = {}+) {+ const bytes = Buffer.alloc(sizeBytes, (index % 251) + 1)+ const transfer = await target.createTransfer({+ callerIdentity: caller,+ callerWorkspacePath: root,+ source: sourceFor(index, options.version ?? 1),+ sizeBytes,+ expiresAt: future(),+ ...(options.owner === undefined ? {} : { retentionOwnerId: options.owner }),+ })+ await nativeFs.writeFile(path.join(root, transfer.partialPath), bytes)+ return { transfer, bytes }+}++async function completedCopy(+ target: GfsDownloadStoreFs,+ root: string,+ caller: string,+ index: number,+ sizeBytes: number,+ options: { version?: number; owner?: string } = {}+) {+ const { transfer, bytes } = await startTransfer(target, root, caller, index, sizeBytes, options)+ const receipt = await target.publish(transfer.id, caller, digest(bytes))+ return { receipt, bytes }+}++async function metricValue(name: string, labels: Record<string, string>): Promise<number> {+ const metric = register.getSingleMetric(name)+ if (metric === undefined) throw new Error(`metric ${name} is not registered`)+ const { values } = await metric.get()+ return values+ .filter(sample => Object.entries(labels).every(([key, value]) => sample.labels[key] === value))+ .reduce((sum, sample) => sum + sample.value, 0)+}++const expiryCount = (outcome: string) =>+ metricValue('clerum_gfs_download_expiry_total', { outcome })+const quotaCount = (scope: string, reason: string) =>+ metricValue('clerum_gfs_download_quota_total', { scope, reason })++async function openStore(host: string): Promise<GfsDownloadStoreFs> {+ const opened = new GfsDownloadStoreFs(host)+ extraStores.push(opened)+ await opened.initialize()+ return opened+}++/** Re-imports the store with environment limits; the logger of that copy is not spied. */+async function loadLimitedStore(env: Record<string, string>) {+ const saved = { ...process.env }+ Object.assign(process.env, env)+ try {+ vi.resetModules()+ const storeModule = await import('./gfsDownloadStoreFs')+ const policyModule = await import('./gfsFilePolicy')+ return { Store: storeModule.GfsDownloadStoreFs, limits: policyModule.GFS_FILE_LIMITS }+ } finally {+ for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]+ for (const [key, value] of Object.entries(saved)) {+ if (value === undefined) delete process.env[key]+ else process.env[key] = value+ }+ }+}++async function limitedStore(env: Record<string, string>) {+ const { Store, limits } = await loadLimitedStore(env)+ const limited = new Store(hostRoot)+ extraStores.push(limited as unknown as GfsDownloadStoreFs)+ await limited.initialize()+ return { limited, limits }+}++/** Distinct createdAt values so the oldest-first eviction order is deterministic. */+function tick(): void {+ vi.setSystemTime(Date.now() + 1_000)+}++async function writeMeta(directory: string, meta: Record<string, unknown>): Promise<void> {+ await nativeFs.writeFile(path.join(directory, 'meta.json'), JSON.stringify(meta), {+ mode: 0o600,+ })+}++function metaFor(id: string, bytes: Buffer, overrides: Record<string, unknown> = {}) {+ const now = Date.now()+ return {+ schemaVersion: 1,+ id,+ callerIdentity: CALLER,+ source: sourceFor(900),+ sizeBytes: bytes.byteLength,+ sha256: digest(bytes),+ createdAt: new Date(now).toISOString(),+ expiresAt: new Date(now + 60 * 60_000).toISOString(),+ ...overrides,+ }+}++async function handMadeDirectory(root: string): Promise<{ id: string; directory: string }> {+ const id = nativeCrypto.randomUUID()+ const directory = downloadDirectory(root, id)+ await nativeFs.mkdir(directory, { recursive: true, mode: 0o700 })+ await nativeFs.chmod(path.dirname(directory), 0o700)+ return { id, directory }+}++beforeEach(async () => {+ nativeFs = await vi.importActual<typeof fs>('node:fs/promises')+ nativeCrypto = await vi.importActual<typeof crypto>('node:crypto')+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))+ for (const boundary of [statfsBoundary, renameBoundary, rmBoundary, lstatBoundary, hashBoundary])+ boundary.mockReset()+ statfsBoundary.mockImplementation(nativeFs.statfs)+ renameBoundary.mockImplementation(nativeFs.rename)+ rmBoundary.mockImplementation(nativeFs.rm)+ lstatBoundary.mockImplementation(nativeFs.lstat)+ hashBoundary.mockImplementation((algorithm: string) => nativeCrypto.createHash(algorithm))+ hostRoot = await nativeFs.mkdtemp(path.join(tmpdir(), 'gfs-store-fs-'))+ callerRoot = await callerDirectory(hostRoot, CALLER)+ store = new GfsDownloadStoreFs(hostRoot)+ await store.initialize()+})++afterEach(async () => {+ vi.restoreAllMocks()+ vi.useRealTimers()+ for (const opened of extraStores.splice(0)) await opened.close(0)+ await store.close(0)+ await nativeFs.rm(hostRoot, { recursive: true, force: true })+})++describe('filesystem-backed GFS download store: publication', () => {+ it('F1: publish writes meta.json before source and every visible source has a meta.json', async () => {+ const { transfer, bytes } = await startTransfer(store, callerRoot, CALLER, 1, 32)+ renameBoundary.mockClear()+ const receipt = await store.publish(transfer.id, CALLER, digest(bytes))++ const renamedTo = renameBoundary.mock.calls.map(([, to]) => path.basename(String(to)))+ // Witness: both renames of the publication happened.+ expect(renamedTo).toHaveLength(2)+ expect(renamedTo).toEqual(['meta.json', 'source'])+ const directory = downloadDirectory(callerRoot, receipt.id)+ expect((await nativeFs.readdir(directory)).sort()).toEqual(['meta.json', 'source'])+ const meta = JSON.parse(await nativeFs.readFile(path.join(directory, 'meta.json'), 'utf8'))+ expect(meta).toMatchObject({+ schemaVersion: 1,+ id: receipt.id,+ callerIdentity: CALLER,+ sizeBytes: 32,+ sha256: digest(bytes),+ })+ expect(receipt.path).toBe(`.gfs-downloads/input-${receipt.id}/source`)+ })++ it('F2: a cancelled publish leaves an incomplete directory that is not reusable and is swept', async () => {+ const { transfer, bytes } = await startTransfer(store, callerRoot, CALLER, 2, 16)+ const controller = new AbortController()+ renameBoundary.mockImplementation(async (from: string, to: string) => {+ await nativeFs.rename(from, to)+ // Cancelled after meta.json is visible and before source is.+ if (path.basename(to) === 'meta.json') controller.abort()+ })+ await expect(+ store.publish(transfer.id, CALLER, digest(bytes), { signal: controller.signal })+ ).rejects.toMatchObject({ code: 'publication_cancelled' })+ renameBoundary.mockImplementation(nativeFs.rename)++ const directory = downloadDirectory(callerRoot, transfer.id)+ // Witness: the cancelled directory is on disk with meta.json and no source.+ expect((await nativeFs.readdir(directory)).sort()).toEqual(['meta.json', 'source.partial'])+ await expect(store.reusableReceipt(CALLER, sourceFor(2), 16)).resolves.toBeUndefined()++ // A Host stop leaves the reservation behind; the next start sweeps it.+ await store.close(0)+ const info = vi.spyOn(logger, 'info')+ await openStore(hostRoot)+ expect(await exists(directory)).toBe(false)+ expect(info).toHaveBeenCalledWith(+ expect.objectContaining({ component: 'GfsDownloadStore', removedIncomplete: 1 }),+ 'GFS download store initialized'+ )+ })++ it('F3: a crash between the two renames leaves meta.json and source.partial and the next store removes it', async () => {+ const bytes = Buffer.alloc(24, 7)+ const { id, directory } = await handMadeDirectory(callerRoot)+ await writeMeta(directory, metaFor(id, bytes))+ await nativeFs.writeFile(path.join(directory, 'source.partial'), bytes, { mode: 0o600 })+ const info = vi.spyOn(logger, 'info')++ await openStore(hostRoot)++ expect(await exists(directory)).toBe(false)+ expect(info).toHaveBeenCalledWith(+ expect.objectContaining({ removedIncomplete: 1, retainedCompleted: 0 }),+ 'GFS download store initialized'+ )+ })++ it('F4: an EIO while writing meta.json fails that publish only and the next admission succeeds', async () => {+ const { transfer, bytes } = await startTransfer(store, callerRoot, CALLER, 4, 16)+ renameBoundary.mockImplementationOnce(async () => {+ throw Object.assign(new Error('injected EIO'), { code: 'EIO' })+ })+ await expect(store.publish(transfer.id, CALLER, digest(bytes))).rejects.toMatchObject({+ code: 'storage_write_failed',+ })+ // The producer's cleanup path: the id stayed active so fail() can remove it.+ await store.fail(transfer.id, CALLER)+ expect(await exists(downloadDirectory(callerRoot, transfer.id))).toBe(false)++ const next = await completedCopy(store, callerRoot, CALLER, 5, 16)+ expect(next.receipt.sha256).toBe(digest(next.bytes))+ await expect(store.readManagedFile(next.receipt.path, CALLER)).resolves.toEqual(next.bytes)+ })+})++describe('filesystem-backed GFS download store: reuse', () => {+ it('F5: reuse returns the same id when the content hash matches', async () => {+ const { receipt } = await completedCopy(store, callerRoot, CALLER, 6, 40)+ const reused = await store.reusableReceipt(CALLER, sourceFor(6), 40)+ expect(reused).toMatchObject({ id: receipt.id, sha256: receipt.sha256, path: receipt.path })+ })++ it('F6: reuse deletes a directory whose content no longer matches and returns undefined', async () => {+ const { receipt } = await completedCopy(store, callerRoot, CALLER, 7, 40)+ const directory = downloadDirectory(callerRoot, receipt.id)+ await nativeFs.writeFile(path.join(directory, 'source'), Buffer.alloc(40, 0xee))+ hashBoundary.mockClear()++ await expect(store.reusableReceipt(CALLER, sourceFor(7), 40)).resolves.toBeUndefined()++ // Witness: the content was hashed exactly once before the verdict.+ expect(hashBoundary).toHaveBeenCalledTimes(1)+ expect(await exists(directory)).toBe(false)+ })++ it('F7: reuse does not match another caller, another version or an expired entry', async () => {+ const { receipt } = await completedCopy(store, callerRoot, CALLER, 8, 12)+ await expect(store.reusableReceipt('caller-b', sourceFor(8), 12)).resolves.toBeUndefined()+ await expect(store.reusableReceipt(CALLER, sourceFor(8, 2), 12)).resolves.toBeUndefined()+ // Witness: the same parameters with the right caller do match.+ await expect(store.reusableReceipt(CALLER, sourceFor(8), 12)).resolves.toMatchObject({+ id: receipt.id,+ })+ vi.setSystemTime(Date.now() + 2 * 60 * 60_000)+ await expect(store.reusableReceipt(CALLER, sourceFor(8), 12)).resolves.toBeUndefined()+ })+})++/** The smallest limits the policy accepts with two callers' worth of Host storage. */+const HOST_30_CALLER_15 = {+ MCP_HOST_GFS_MAX_FILE_BYTES: '10',+ MCP_HOST_GFS_DOWNLOAD_STORAGE_BYTES: '30',+ MCP_HOST_GFS_CALLER_DOWNLOAD_STORAGE_BYTES: '15',+}++describe('filesystem-backed GFS download store: quotas and eviction', () => {+ it("F8: a caller over quota evicts its own oldest completed downloads, never another caller's", async () => {+ const { limited, limits } = await limitedStore({ MCP_HOST_GFS_CALLER_DOWNLOAD_MAX_FILES: '2' })+ expect(limits.callerRetainedFiles).toBe(2)+ const otherRoot = await callerDirectory(hostRoot, 'caller-b')+ const other = await completedCopy(limited, otherRoot, 'caller-b', 10, 8)+ tick()+ const oldest = await completedCopy(limited, callerRoot, CALLER, 11, 8)+ tick()+ const newer = await completedCopy(limited, callerRoot, CALLER, 12, 8)+ tick()++ await startTransfer(limited, callerRoot, CALLER, 13, 8)++ expect(await exists(downloadDirectory(callerRoot, oldest.receipt.id))).toBe(false)+ // Witnesses: the caller's newer copy and the other caller's older copy stay.+ expect(await exists(downloadDirectory(callerRoot, newer.receipt.id))).toBe(true)+ expect(await exists(downloadDirectory(otherRoot, other.receipt.id))).toBe(true)+ })++ it("F9: a host over quota may evict any caller's oldest completed download", async () => {+ const { limited } = await limitedStore(HOST_30_CALLER_15)+ const otherRoot = await callerDirectory(hostRoot, 'caller-b')+ const thirdRoot = await callerDirectory(hostRoot, 'caller-c')+ const other = await completedCopy(limited, otherRoot, 'caller-b', 20, 10)+ tick()+ const third = await completedCopy(limited, thirdRoot, 'caller-c', 21, 10)+ tick()+ const own = await completedCopy(limited, callerRoot, CALLER, 22, 10)+ tick()++ // caller-a stays within 15; the Host would reach 35 of 30.+ await startTransfer(limited, callerRoot, CALLER, 23, 5)++ expect(await exists(downloadDirectory(otherRoot, other.receipt.id))).toBe(false)+ // Witnesses: only the oldest copy went, whoever owned the newer ones.+ expect(await exists(downloadDirectory(thirdRoot, third.receipt.id))).toBe(true)+ expect(await exists(downloadDirectory(callerRoot, own.receipt.id))).toBe(true)+ })++ it('F10: when eviction cannot make room nothing is deleted', async () => {+ const { limited } = await limitedStore({ MCP_HOST_GFS_CALLER_DOWNLOAD_MAX_FILES: '2' })+ await completedCopy(limited, callerRoot, CALLER, 30, 8, { owner: 'task-1' })+ tick()+ await completedCopy(limited, callerRoot, CALLER, 31, 8, { owner: 'task-1' })+ const downloads = path.join(callerRoot, '.gfs-downloads')+ const before = (await nativeFs.readdir(downloads)).sort()+ const denied = await quotaCount('caller', 'retained_files')+ lstatBoundary.mockClear()++ await expect(startTransfer(limited, callerRoot, CALLER, 32, 8)).rejects.toMatchObject({+ code: 'caller_quota_exceeded',+ })++ // Witness: the admission read the inventory before refusing.+ expect(lstatBoundary.mock.calls.length).toBeGreaterThan(0)+ expect((await nativeFs.readdir(downloads)).sort()).toEqual(before)+ expect(before).toHaveLength(2)+ expect(await quotaCount('caller', 'retained_files')).toBe(denied + 1)+ })++ it('F11: pinned and active downloads are never evicted', async () => {+ const { limited } = await limitedStore({+ ...HOST_30_CALLER_15,+ })+ const otherRoot = await callerDirectory(hostRoot, 'caller-b')+ const thirdRoot = await callerDirectory(hostRoot, 'caller-c')+ const pinned = await completedCopy(limited, callerRoot, CALLER, 40, 10, { owner: 'task-40' })+ tick()+ const active = await startTransfer(limited, otherRoot, 'caller-b', 41, 10)+ tick()+ const unpinned = await completedCopy(limited, thirdRoot, 'caller-c', 42, 10)+ tick()++ // The Host would reach 40 of 30; one eviction makes room.+ await startTransfer(limited, await callerDirectory(hostRoot, 'caller-d'), 'caller-d', 43, 10)++ // Witness: the only eligible copy, although the newest, was evicted.+ expect(await exists(downloadDirectory(thirdRoot, unpinned.receipt.id))).toBe(false)+ expect(await exists(downloadDirectory(callerRoot, pinned.receipt.id))).toBe(true)+ expect(await exists(downloadDirectory(otherRoot, active.transfer.id))).toBe(true)+ })++ it('F12: free space is checked with active reservations and the 16 MiB reserve', async () => {+ const otherRoot = await callerDirectory(hostRoot, 'caller-b')+ await startTransfer(store, otherRoot, 'caller-b', 50, 4096 * 10)+ const observed = await nativeFs.statfs(hostRoot, { bigint: true })+ const reserveBlocks = (16n * 1024n * 1024n) / 4096n+ // Room for the request and the reserve, but not for the active reservation.+ statfsBoundary.mockResolvedValue({ ...observed, bsize: 4096n, bavail: reserveBlocks + 2n })+ const denied = await quotaCount('host', 'free_space')++ await expect(startTransfer(store, callerRoot, CALLER, 51, 4096)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(statfsBoundary).toHaveBeenCalled()+ expect(await quotaCount('host', 'free_space')).toBe(denied + 1)++ // Control: with room for the active reservation too, the same request is admitted.+ statfsBoundary.mockResolvedValue({ ...observed, bsize: 4096n, bavail: reserveBlocks + 11n })+ await expect(startTransfer(store, callerRoot, CALLER, 51, 4096)).resolves.toBeDefined()+ })++ it('F13: eviction candidates are not hashed', async () => {+ const { limited } = await limitedStore({+ ...HOST_30_CALLER_15,+ })+ const first = await completedCopy(limited, callerRoot, CALLER, 60, 3)+ tick()+ const second = await completedCopy(limited, callerRoot, CALLER, 61, 3)+ tick()+ const third = await completedCopy(limited, callerRoot, CALLER, 62, 9)+ tick()+ hashBoundary.mockClear()++ // 15 retained + 6 requested against 15: the plan takes the two 3-byte copies.+ await startTransfer(limited, callerRoot, CALLER, 64, 6)++ // Witness: two candidates were evicted.+ expect(await exists(downloadDirectory(callerRoot, first.receipt.id))).toBe(false)+ expect(await exists(downloadDirectory(callerRoot, second.receipt.id))).toBe(false)+ expect(hashBoundary).not.toHaveBeenCalled()+ expect(await exists(downloadDirectory(callerRoot, third.receipt.id))).toBe(true)+ })++ it('F14: size above maxFileBytes is refused before any filesystem work', async () => {+ const { limited } = await limitedStore({ MCP_HOST_GFS_MAX_FILE_BYTES: '10' })+ statfsBoundary.mockClear()+ lstatBoundary.mockClear()+ await expect(startTransfer(limited, callerRoot, CALLER, 70, 11)).rejects.toMatchObject({+ code: 'host_quota_exceeded',+ })+ expect(statfsBoundary).not.toHaveBeenCalled()+ expect(lstatBoundary).not.toHaveBeenCalled()+ // Witness: the boundaries are live for an admissible size.+ await startTransfer(limited, callerRoot, CALLER, 71, 10)+ expect(statfsBoundary).toHaveBeenCalled()+ expect(lstatBoundary).toHaveBeenCalled()+ })++ it('F15: concurrency limits per caller and per host return download_busy', async () => {+ const callerBusy = await quotaCount('caller', 'active_downloads')+ const hostBusy = await quotaCount('host', 'active_downloads')+ await startTransfer(store, callerRoot, CALLER, 80, 4)+ await expect(startTransfer(store, callerRoot, CALLER, 81, 4)).rejects.toMatchObject({+ code: 'download_busy',+ })+ expect(await quotaCount('caller', 'active_downloads')).toBe(callerBusy + 1)++ await startTransfer(store, await callerDirectory(hostRoot, 'caller-b'), 'caller-b', 82, 4)+ await expect(+ startTransfer(store, await callerDirectory(hostRoot, 'caller-c'), 'caller-c', 83, 4)+ ).rejects.toMatchObject({ code: 'download_busy' })+ expect(await quotaCount('host', 'active_downloads')).toBe(hostBusy + 1)+ })+})++describe('filesystem-backed GFS download store: sweep', () => {+ it('F16: the sweep removes incomplete directories of every shape and keeps complete ones', async () => {+ const kept = await completedCopy(store, callerRoot, CALLER, 90, 20)+ const bytes = Buffer.alloc(20, 3)+ const shapes: string[] = []++ const orphanPartial = await handMadeDirectory(callerRoot)+ await nativeFs.writeFile(path.join(orphanPartial.directory, 'source.partial'), bytes)+ const metaWithoutSource = await handMadeDirectory(callerRoot)+ await writeMeta(metaWithoutSource.directory, metaFor(metaWithoutSource.id, bytes))+ const sourceWithoutMeta = await handMadeDirectory(callerRoot)+ await nativeFs.writeFile(path.join(sourceWithoutMeta.directory, 'source'), bytes)+ const invalidMeta = await handMadeDirectory(callerRoot)+ await writeMeta(invalidMeta.directory, metaFor(invalidMeta.id, bytes, { sha256: 'nope' }))+ await nativeFs.writeFile(path.join(invalidMeta.directory, 'source'), bytes)+ const wrongSize = await handMadeDirectory(callerRoot)+ await writeMeta(wrongSize.directory, metaFor(wrongSize.id, bytes))+ await nativeFs.writeFile(path.join(wrongSize.directory, 'source'), Buffer.alloc(21, 3))+ for (const shape of [+ orphanPartial,+ metaWithoutSource,+ sourceWithoutMeta,+ invalidMeta,+ wrongSize,+ ])+ shapes.push(shape.directory)+ const removed = await expiryCount('incomplete_removed')++ await expect(store.cleanupExpired()).resolves.toEqual({+ removedExpired: 0,+ removedIncomplete: 5,+ removeFailed: 0,+ })++ for (const directory of shapes) expect(await exists(directory)).toBe(false)+ expect(await expiryCount('incomplete_removed')).toBe(removed + 5)+ // Witness: the complete download stays on disk and readable through the index.+ await expect(store.readManagedFile(kept.receipt.path, CALLER)).resolves.toEqual(kept.bytes)+ })++ it('F17: the sweep removes expired complete downloads unless pinned', async () => {+ const pinned = await completedCopy(store, callerRoot, CALLER, 100, 8, { owner: 'task-100' })+ const loose = await completedCopy(store, callerRoot, CALLER, 101, 8)+ const later = Date.now() + 2 * 60 * 60_000++ await expect(store.cleanupExpired(later)).resolves.toEqual({+ removedExpired: 1,+ removedIncomplete: 0,+ removeFailed: 0,+ })+ expect(await exists(downloadDirectory(callerRoot, loose.receipt.id))).toBe(false)+ expect(await exists(downloadDirectory(callerRoot, pinned.receipt.id))).toBe(true)++ await store.releaseReceiptOwner('task-100', CALLER)+ await expect(store.cleanupExpired(later)).resolves.toMatchObject({ removedExpired: 1 })+ expect(await exists(downloadDirectory(callerRoot, pinned.receipt.id))).toBe(false)+ })++ it('F18: the sweep ignores entries that are not input-<uuid>', async () => {+ const downloads = path.join(callerRoot, '.gfs-downloads')+ await nativeFs.mkdir(downloads, { mode: 0o700 })+ const foreign = [+ path.join(downloads, 'input-not-a-uuid'),+ path.join(downloads, `input-${nativeCrypto.randomUUID().toUpperCase()}`),+ path.join(downloads, 'notes'),+ ]+ for (const directory of foreign) await nativeFs.mkdir(directory, { mode: 0o700 })+ await nativeFs.writeFile(path.join(downloads, 'readme.txt'), 'kept')+ const incomplete = await handMadeDirectory(callerRoot)++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ // Witness: the incomplete store directory next to them was removed.+ expect(await exists(incomplete.directory)).toBe(false)+ for (const directory of foreign) expect(await exists(directory)).toBe(true)+ expect(await exists(path.join(downloads, 'readme.txt'))).toBe(true)+ })++ it('F19: a symlinked .gfs-downloads is removed without following it', async () => {+ const outside = await nativeFs.mkdtemp(path.join(tmpdir(), 'gfs-store-fs-outside-'))+ try {+ const victim = path.join(outside, `input-${nativeCrypto.randomUUID()}`)+ await nativeFs.mkdir(victim, { mode: 0o700 })+ await nativeFs.writeFile(path.join(victim, 'source.partial'), 'outside')+ const link = path.join(callerRoot, '.gfs-downloads')+ await nativeFs.symlink(outside, link)++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(await exists(link)).toBe(false)+ await expect(nativeFs.readFile(path.join(victim, 'source.partial'), 'utf8')).resolves.toBe(+ 'outside'+ )+ } finally {+ await nativeFs.rm(outside, { recursive: true, force: true })+ }+ })++ it('F20: a failed removal is logged and counted, does not block admission and is retried', async () => {+ const stuck = await handMadeDirectory(callerRoot)+ rmBoundary.mockImplementation(async (target: string, options: unknown) => {+ // The store renames the directory to a private `.trash-<uuid>` name, then removes that.+ if (path.basename(target).startsWith('.trash-'))+ throw Object.assign(new Error('injected EACCES'), { code: 'EACCES' })+ return nativeFs.rm(target, options as Parameters<typeof nativeFs.rm>[1])+ })+ const warn = vi.spyOn(logger, 'warn')+ const failed = await expiryCount('remove_failed')++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removeFailed: 1 })++ expect(warn).toHaveBeenCalledWith(+ { component: 'GfsDownloadStore', code: 'EACCES' },+ expect.stringContaining('could not remove')+ )+ expect(await expiryCount('remove_failed')).toBe(failed + 1)+ const downloads = path.dirname(stuck.directory)+ const leftover = (await nativeFs.readdir(downloads)).filter(name => name.startsWith('.trash-'))+ expect(leftover).toHaveLength(1)+ expect(await exists(stuck.directory)).toBe(false)+ const next = await completedCopy(store, callerRoot, CALLER, 110, 8)+ await expect(store.readManagedFile(next.receipt.path, CALLER)).resolves.toEqual(next.bytes)++ // Once removal works again, the next sweep removes the renamed leftover.+ rmBoundary.mockImplementation(nativeFs.rm)+ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })+ expect(await exists(path.join(downloads, leftover[0]!))).toBe(false)+ })+})++describe('filesystem-backed GFS download store: managed reads', () => {+ it('F21: readManagedFile returns the exact bytes and answers another caller like a missing copy', async () => {+ const { receipt, bytes } = await completedCopy(store, callerRoot, CALLER, 120, 33)+ await expect(store.readManagedFile(receipt.path, CALLER)).resolves.toEqual(bytes)+ await expect(store.readManagedFile(receipt.path, 'caller-b')).rejects.toMatchObject({+ code: 'download_missing',+ })+ await expect(+ store.readManagedFile(`.gfs-downloads/input-${nativeCrypto.randomUUID()}/source`, CALLER)+ ).rejects.toMatchObject({ code: 'download_missing' })+ await expect(store.readManagedFile('../escape/source', CALLER)).rejects.toMatchObject({+ code: 'download_missing',+ })+ })++ it('F22: readManagedFile deletes a tampered source and reports download_missing', async () => {+ const { receipt } = await completedCopy(store, callerRoot, CALLER, 122, 16)+ const directory = downloadDirectory(callerRoot, receipt.id)+ await nativeFs.writeFile(path.join(directory, 'source'), Buffer.alloc(16, 0xab))+ hashBoundary.mockClear()++ await expect(store.readManagedFile(receipt.path, CALLER)).rejects.toMatchObject({+ code: 'download_missing',+ })++ // Witness: the bytes were hashed once, then the directory was removed.+ expect(hashBoundary).toHaveBeenCalledTimes(1)+ expect(await exists(directory)).toBe(false)+ })++ it('F23: readManagedFilePrefix bounds and size check', async () => {+ const bytes = Buffer.from('0123456789abcdef0123')+ const transfer = await store.createTransfer({+ callerIdentity: CALLER,+ callerWorkspacePath: callerRoot,+ source: sourceFor(123),+ sizeBytes: bytes.byteLength,+ expiresAt: future(),+ })+ await nativeFs.writeFile(path.join(callerRoot, transfer.partialPath), bytes)+ const receipt = await store.publish(transfer.id, CALLER, digest(bytes))++ await expect(store.readManagedFilePrefix(receipt.path, CALLER, 4)).resolves.toEqual(+ Buffer.from('0123')+ )+ await expect(store.readManagedFilePrefix(receipt.path, CALLER)).resolves.toEqual(+ bytes.subarray(0, 16)+ )+ for (const invalid of [0, -1, 4097, 1.5])+ await expect(store.readManagedFilePrefix(receipt.path, CALLER, invalid)).rejects.toThrow(+ RangeError+ )+ await nativeFs.appendFile(path.join(downloadDirectory(callerRoot, receipt.id), 'source'), 'x')+ await expect(store.readManagedFilePrefix(receipt.path, CALLER, 4)).rejects.toMatchObject({+ code: 'download_missing',+ })+ })+})++describe('filesystem-backed GFS download store: pins, lifecycle and errors', () => {+ it('F24: releaseReceiptOwner unpins only its own caller and a second store on the same root sees no pins', async () => {+ const pinned = await completedCopy(store, callerRoot, CALLER, 130, 8, { owner: 'task-130' })+ const rootB = await callerDirectory(hostRoot, 'caller-b')+ // The same owner id under another caller is another pin, not a refusal.+ const other = await completedCopy(store, rootB, 'caller-b', 131, 8, { owner: 'task-130' })+ const later = Date.now() + 2 * 60 * 60_000+ await expect(store.releaseReceiptOwner('unknown-owner', CALLER)).resolves.toBeUndefined()+ await expect(store.releaseReceiptOwner('task-130', 'caller-b')).resolves.toBeUndefined()+ // Witness: caller-b's release freed only caller-b's copy; CALLER's pin holds.+ await expect(store.cleanupExpired(later)).resolves.toMatchObject({ removedExpired: 1 })+ expect(await exists(downloadDirectory(rootB, other.receipt.id))).toBe(false)+ expect(await exists(downloadDirectory(callerRoot, pinned.receipt.id))).toBe(true)++ await store.close()+ const second = await openStore(hostRoot)+ await expect(second.cleanupExpired(later)).resolves.toMatchObject({ removedExpired: 1 })+ expect(await exists(downloadDirectory(callerRoot, pinned.receipt.id))).toBe(false)+ })++ it('F25: close drains active transfers and refuses new admissions', async () => {+ const { transfer, bytes } = await startTransfer(store, callerRoot, CALLER, 140, 8)+ let closed = false+ const closing = store.close().then(() => {+ closed = true+ })+ await expect(+ startTransfer(store, await callerDirectory(hostRoot, 'caller-b'), 'caller-b', 141, 8)+ ).rejects.toMatchObject({ code: 'download_busy' })+ expect(store.isAvailable()).toBe(false)+ expect(closed).toBe(false)++ const receipt = await store.publish(transfer.id, CALLER, digest(bytes))+ await closing+ expect(closed).toBe(true)+ expect(receipt.id).toBe(transfer.id)+ await expect(store.initialize()).rejects.toMatchObject({ code: 'download_busy' })+ })++ it('F25b: close with an active transfer that never finishes resolves within the deadline', async () => {+ const { transfer, bytes } = await startTransfer(store, callerRoot, CALLER, 150, 8)+ const warn = vi.spyOn(logger, 'warn')+ const started = performance.now()++ await store.close(50)++ expect(performance.now() - started).toBeLessThan(2_000)+ // Witness: the store reports the transfer it left behind.+ expect(warn).toHaveBeenCalledWith(+ { component: 'GfsDownloadStore', active: 1 },+ expect.stringContaining('closed with active transfers')+ )+ await expect(store.publish(transfer.id, CALLER, digest(bytes))).rejects.toMatchObject({+ code: 'download_busy',+ })+ await expect(store.fail(transfer.id, CALLER)).rejects.toMatchObject({ code: 'download_busy' })+ await expect(store.close()).resolves.toBeUndefined()+ })++ it('F26: error codes for an uninitialized, a closed and an unknown transfer', async () => {+ const uninitialized = new GfsDownloadStoreFs(hostRoot)+ const probe = `.gfs-downloads/input-${nativeCrypto.randomUUID()}/source`+ await expect(uninitialized.readManagedFile(probe, CALLER)).rejects.toMatchObject({+ code: 'workspace_unavailable',+ })+ await expect(+ uninitialized.createTransfer({+ callerIdentity: CALLER,+ callerWorkspacePath: callerRoot,+ source: sourceFor(160),+ sizeBytes: 1,+ expiresAt: future(),+ })+ ).rejects.toMatchObject({ code: 'workspace_unavailable' })+ await expect(uninitialized.close()).resolves.toBeUndefined()+ await expect(uninitialized.initialize()).rejects.toMatchObject({ code: 'download_busy' })++ const { transfer } = await startTransfer(store, callerRoot, CALLER, 161, 4)+ // Another caller's transfer is answered exactly like an unknown id.+ await expect(store.fail(transfer.id, 'caller-b')).rejects.toMatchObject({+ code: 'download_busy',+ })+ await expect(store.fail(nativeCrypto.randomUUID(), CALLER)).rejects.toMatchObject({+ code: 'download_busy',+ })+ await expect(+ store.publish(nativeCrypto.randomUUID(), CALLER, 'a'.repeat(64))+ ).rejects.toMatchObject({ code: 'download_missing' })+ await expect(store.publish(transfer.id, CALLER, 'not-a-digest')).rejects.toThrow(RangeError)+ // Witness: the transfer is live and the caller's fail() removes it.+ await store.fail(transfer.id, CALLER)+ expect(await exists(downloadDirectory(callerRoot, transfer.id))).toBe(false)++ await store.close()+ await expect(store.readManagedFile(probe, CALLER)).rejects.toMatchObject({+ code: 'download_busy',+ })+ await expect(store.close()).resolves.toBeUndefined()+ })++ it('F27: a Host root that is a symlink makes initialize reject with workspace_unavailable', async () => {+ const real = path.join(hostRoot, 'real-store')+ await nativeFs.mkdir(real, { mode: 0o700 })+ const link = path.join(hostRoot, 'store-link')+ await nativeFs.symlink(real, link)+ const linked = new GfsDownloadStoreFs(link)+ extraStores.push(linked)++ await expect(linked.initialize()).rejects.toBeInstanceOf(GfsDownloadStoreError)+ await expect(linked.initialize()).rejects.toMatchObject({ code: 'workspace_unavailable' })+ expect(linked.isAvailable()).toBe(false)+ // Witness: the same directory through its real path initializes.+ await expect(openStore(real)).resolves.toBeInstanceOf(GfsDownloadStoreFs)+ })+})diff --git a/mcp-host/src/internalTools/gfsDownloadStoreFs.ts b/mcp-host/src/internalTools/gfsDownloadStoreFs.tsnew file mode 100644index 000000000..6f5c3b794--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreFs.ts@@ -0,0 +1,1409 @@+import { createHash, randomUUID } from 'node:crypto'+import { constants } from 'node:fs'+import * as fs from 'node:fs/promises'+import * as path from 'node:path'+import { logger } from '../logger'+import type { GfsImageSource } from '../visualInput/policy'+import { recordGfsDownloadExpiry, recordGfsDownloadQuota } from './gfsDownloadMetrics'+import {+ GFS_FILE_LIMITS,+ GFS_HOST_ACTIVE_DOWNLOADS,+ GFS_HOST_RETAINED_FILES,+} from './gfsFilePolicy'+import { openPrivateStoreObject, verifyPrivateStoreDirectory } from './gfsStorePrivateFiles'++const GFS_DOWNLOAD_STORE_LOG_COMPONENT = 'GfsDownloadStore'++/** Caller-root directory that holds every download of that caller. */+const DOWNLOADS_DIRECTORY = '.gfs-downloads'+const META_FILE = 'meta.json'+const SOURCE_FILE = 'source'+const PARTIAL_FILE = 'source.partial'+const META_MAX_BYTES = 64 * 1024+const FREE_SPACE_RESERVE_BYTES = 16n * 1024n * 1024n++const UUID_PATTERN = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'+/** The only directory names under `.gfs-downloads` that belong to the store. */+export const GFS_INPUT_DIRECTORY_RE = new RegExp(`^input-(${UUID_PATTERN})$`)+/** A directory renamed out of the way before removal; swept if a removal stops. */+const TRASH_PREFIX = '.trash-'+const TRASH_DIRECTORY_RE = new RegExp(`^\\.trash-${UUID_PATTERN}$`)+/** Caller-relative receipt path; the same contract gfsFilePreparation enforces. */+const RECEIPT_PATH_RE = new RegExp(`^\\.gfs-downloads/input-(${UUID_PATTERN})/source$`)+const SHA256_RE = /^[0-9a-f]{64}$/++export type GfsDownloadStoreErrorCode =+ | 'caller_mismatch'+ | 'caller_quota_exceeded'+ | 'download_busy'+ | 'download_expired'+ | 'download_missing'+ | 'host_quota_exceeded'+ | 'publication_cancelled'+ | 'storage_write_failed'+ | 'workspace_unavailable'++export class GfsDownloadStoreError extends Error {+ constructor(readonly code: GfsDownloadStoreErrorCode) {+ super(`GFS download store failed (${code})`)+ this.name = 'GfsDownloadStoreError'+ }+}++export interface GfsDownloadTransfer {+ id: string+ path: string+ partialPath: string+ sizeBytes: number+ expiresAt: string+}++export interface GfsDownloadReceipt {+ id: string+ source: GfsImageSource+ path: string+ sizeBytes: number+ sha256: string+ expiresAt: string+}++/** Serialized receipt written to `meta.json`; the store writes nothing else. */+interface StoredMeta {+ schemaVersion: 1+ id: string+ callerIdentity: string+ source: GfsImageSource+ sizeBytes: number+ sha256: string+ createdAt: string+ expiresAt: string+}++/**+ * A complete download: `meta.json` parses and `source` has the recorded size.+ * Provenance lives only in memory. A `published` entry was written by this+ * process, so its caller identity and sha256 are trusted. An `adopted` entry+ * was found on disk (left by an earlier process or planted by a shell command+ * running with the Host UID): it counts against the quota of the directory+ * that contains it and is swept, but it is never reused, read or pinned.+ */+/** dev/ino of an inode, compared against an open descriptor or a later lstat. */+interface InodeIdentity {+ dev: bigint+ ino: bigint+}++function sameInode(left: InodeIdentity, right: InodeIdentity): boolean {+ return left.dev === right.dev && left.ino === right.ino+}++interface Entry extends StoredMeta {+ /** Absolute `<callerRoot>/.gfs-downloads/input-<id>` directory. */+ directory: string+ /**+ * The `source` inode this process published. Reads and reuse serve only a+ * descriptor whose inode is this one; adopted entries have none and are+ * never served.+ */+ sourceIdentity?: InodeIdentity+ /** `<hostRoot>/users/<key>` containing the entry: the owner for quota and inventory. */+ callerRoot: string+ provenance: 'published' | 'adopted'+ /** Last publish, reuse or managed read in this process; adopted entries use createdAt. */+ lastUsedMs: number+}++/** A reservation held in memory between createTransfer and publish/fail. */+interface ActiveTransfer {+ id: string+ callerIdentity: string+ callerRoot: string+ source: GfsImageSource+ directory: string+ /** The input directory created at admission; publication writes only into it. */+ directoryIdentity?: InodeIdentity+ sizeBytes: number+ createdAt: string+ expiresAt: string+}++export interface GfsDownloadSweepResult {+ removedExpired: number+ removedIncomplete: number+ removeFailed: number+}++interface SweepTotals extends GfsDownloadSweepResult {+ retainedCompleted: number+ retainedBytes: number+ /** Retained entries this process did not publish. */+ adopted: number+}++export interface GfsDownloadInventory {+ bytes: number+ files: number+ /** Keyed by the caller directory name (`users/<key>`), never by a recorded identity. */+ byCaller: Map<string, { bytes: number; files: number }>+}++type DirectoryState =+ | { state: 'absent' }+ | { state: 'incomplete' }+ | { state: 'complete'; entry: Entry }++function isValidSource(value: unknown): value is GfsImageSource {+ if (typeof value !== 'object' || value === null || Array.isArray(value)) return false+ const source = value as Record<string, unknown>+ return (+ source.kind === 'gfs' &&+ typeof source.drive === 'string' &&+ typeof source.resourceId === 'string' &&+ typeof source.gfsUri === 'string' &&+ typeof source.name === 'string' &&+ Number.isSafeInteger(source.version) &&+ (source.version as number) >= 0 &&+ (source.attachmentId === undefined || typeof source.attachmentId === 'string') &&+ (source.toolCallId === undefined || typeof source.toolCallId === 'string')+ )+}++/** A recorded createdAt may lead this clock by at most this much (clock skew between Pods). */+const CREATED_AT_MAX_LEAD_MS = 60_000++/**+ * Returns undefined for anything that is not a schema-1 receipt for+ * `expectedId`. The time bounds keep a planted meta.json from outliving the+ * retention window: no complete entry can expire later than+ * `now + CREATED_AT_MAX_LEAD_MS + retentionMs`.+ */+function parseMeta(raw: string, expectedId: string, now: number): StoredMeta | undefined {+ let parsed: unknown+ try {+ parsed = JSON.parse(raw)+ } catch {+ return undefined+ }+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return undefined+ const meta = parsed as Record<string, unknown>+ const createdAt = typeof meta.createdAt === 'string' ? Date.parse(meta.createdAt) : NaN+ const expiresAt = typeof meta.expiresAt === 'string' ? Date.parse(meta.expiresAt) : NaN+ if (+ meta.schemaVersion !== 1 ||+ meta.id !== expectedId ||+ typeof meta.callerIdentity !== 'string' ||+ meta.callerIdentity.length === 0 ||+ !isValidSource(meta.source) ||+ !Number.isSafeInteger(meta.sizeBytes) ||+ (meta.sizeBytes as number) < 0 ||+ (meta.sizeBytes as number) > GFS_FILE_LIMITS.maxFileBytes ||+ typeof meta.sha256 !== 'string' ||+ !SHA256_RE.test(meta.sha256) ||+ !Number.isFinite(createdAt) ||+ !Number.isFinite(expiresAt) ||+ expiresAt < createdAt ||+ expiresAt > createdAt + GFS_FILE_LIMITS.retentionMs ||+ createdAt > now + CREATED_AT_MAX_LEAD_MS+ )+ return undefined+ return meta as unknown as StoredMeta+}++function validReceiptOwnerId(ownerId: string): boolean {+ return (+ typeof ownerId === 'string' &&+ ownerId.length > 0 &&+ ownerId.length <= 256 &&+ !/[\u0000-\u001f\u007f]/.test(ownerId) &&+ !['__proto__', 'constructor', 'prototype'].includes(ownerId)+ )+}++function isWithinDirectory(child: string, parent: string): boolean {+ const relative = path.relative(parent, child)+ return relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative)+}++function errorCode(error: unknown): string {+ const code = (error as NodeJS.ErrnoException)?.code+ return typeof code === 'string' ? code : 'unknown'+}++/** A removal is proven by ENOENT, not by `fs.rm` returning. */+async function assertAbsent(target: string): Promise<void> {+ try {+ await fs.lstat(target)+ } catch (error) {+ if (errorCode(error) === 'ENOENT') return+ throw error+ }+ throw new GfsDownloadStoreError('storage_write_failed')+}++async function syncDirectory(directory: string): Promise<void> {+ const handle = await fs.open(directory, 'r')+ try {+ await handle.sync()+ } finally {+ await handle.close()+ }+}++async function sha256FileHandle(handle: fs.FileHandle, assertActive?: () => void): Promise<string> {+ const digest = createHash('sha256')+ const chunk = Buffer.alloc(64 * 1024)+ let position = 0+ for (;;) {+ assertActive?.()+ const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, position)+ if (bytesRead === 0) break+ digest.update(chunk.subarray(0, bytesRead))+ position += bytesRead+ }+ assertActive?.()+ return digest.digest('hex')+}++function receiptPath(id: string): string {+ return path.join(DOWNLOADS_DIRECTORY, `input-${id}`, SOURCE_FILE)+}++/** `<callerRoot>/.gfs-downloads/input-<id>` → `<callerRoot>`. */+/** Owner ids are scoped to their caller; the encoding is unambiguous for any strings. */+function pinKey(callerIdentity: string, ownerId: string): string {+ return JSON.stringify([callerIdentity, ownerId])+}++function callerRootOf(directory: string): string {+ return path.dirname(path.dirname(directory))+}++/** Adopted copies go first (oldest recorded first), then published ones least recently used. */+function evictionOrder(left: Entry, right: Entry): number {+ if (left.provenance !== right.provenance) return left.provenance === 'adopted' ? -1 : 1+ return left.lastUsedMs - right.lastUsedMs || left.id.localeCompare(right.id)+}++/**+ * GFS download store whose only state is the directory tree under+ * `users/<caller>/.gfs-downloads`. The filesystem is the truth for existence,+ * quota and cleanup: the in-memory index is rebuilt from disk at startup and+ * re-checked on admission and by every sweep. It is never the truth for who+ * downloaded a copy or for its hash: reuse and managed reads serve only copies+ * this process published. Active transfers and retention pins live in memory.+ */+export class GfsDownloadStoreFs {+ private hostRoot: string+ private readonly requestedHostRoot: string+ private readonly entries = new Map<string, Entry>()+ private readonly active = new Map<string, ActiveTransfer>()+ private readonly activeByCaller = new Map<string, number>()+ /** Keyed by `pinKey(callerIdentity, ownerId)`: callers never share or see a pin. */+ private readonly pins = new Map<string, Set<string>>()+ private mutationTail: Promise<void> = Promise.resolve()+ private drainWaiters: Array<() => void> = []+ private initializing?: Promise<void>+ private initialized = false+ private closing = false+ private closed = false++ constructor(hostRoot: string) {+ this.hostRoot = path.resolve(hostRoot)+ this.requestedHostRoot = this.hostRoot+ }++ isAvailable(): boolean {+ return this.initialized && !this.closing && !this.closed+ }++ /**+ * Rejects only when the Host root itself is unusable (workspace_unavailable)+ * or the store was closed (download_busy). Leftover download directories+ * are removed or adopted; none of them can make initialize fail.+ */+ async initialize(): Promise<void> {+ if (this.closed || this.closing) throw new GfsDownloadStoreError('download_busy')+ if (this.initialized) return+ this.initializing ??= this.runInitialize().finally(() => {+ this.initializing = undefined+ })+ return this.initializing+ }++ private async runInitialize(): Promise<void> {+ try {+ await fs.mkdir(this.requestedHostRoot, { recursive: true, mode: 0o700 })+ const requested = await fs.lstat(this.requestedHostRoot)+ if (!requested.isDirectory() || requested.isSymbolicLink())+ throw new GfsDownloadStoreError('workspace_unavailable')+ this.hostRoot = await fs.realpath(this.requestedHostRoot)+ const resolved = await fs.lstat(this.hostRoot)+ if (!resolved.isDirectory() || resolved.isSymbolicLink())+ throw new GfsDownloadStoreError('workspace_unavailable')+ } catch (error) {+ if (error instanceof GfsDownloadStoreError) throw error+ throw new GfsDownloadStoreError('workspace_unavailable')+ }+ const totals = await this.serialize(() => this.sweep(Date.now()))+ if (this.closed || this.closing) throw new GfsDownloadStoreError('download_busy')+ this.initialized = true+ logger.info(+ {+ component: GFS_DOWNLOAD_STORE_LOG_COMPONENT,+ removedIncomplete: totals.removedIncomplete,+ removedExpired: totals.removedExpired,+ retainedCompleted: totals.retainedCompleted,+ retainedBytes: totals.retainedBytes,+ adopted: totals.adopted,+ retiredLegacyStore: false,+ },+ 'GFS download store initialized'+ )+ }++ async createTransfer(input: {+ callerIdentity: string+ callerWorkspacePath: string+ source: GfsImageSource+ sizeBytes: number+ expiresAt: string+ /** Task lifetime protection; release only after every physical consumer has settled. */+ retentionOwnerId?: string+ }): Promise<GfsDownloadTransfer> {+ this.assertAdmitting()+ if (+ !Number.isSafeInteger(input.sizeBytes) ||+ input.sizeBytes < 0 ||+ input.sizeBytes > GFS_FILE_LIMITS.maxFileBytes+ )+ throw new GfsDownloadStoreError('host_quota_exceeded')+ if (typeof input.callerIdentity !== 'string' || input.callerIdentity.length === 0)+ throw new GfsDownloadStoreError('caller_mismatch')+ if (!isValidSource(input.source)) throw new RangeError('GFS download source is malformed')+ const createdAtMs = Date.now()+ const expiresAtMs = Date.parse(input.expiresAt)+ if (!Number.isFinite(expiresAtMs) || expiresAtMs < createdAtMs)+ throw new RangeError('GFS download expiry must be a future ISO timestamp')+ // The same bound parseMeta applies, so a published copy stays valid on disk.+ if (expiresAtMs > createdAtMs + GFS_FILE_LIMITS.retentionMs)+ throw new RangeError('GFS download expiry must be within the retention window')+ if (input.retentionOwnerId !== undefined) this.assertReceiptOwner(input.retentionOwnerId)+ const callerRoot = await this.validateCallerRoot(input.callerWorkspacePath)+ const id = randomUUID()+ const downloadsRoot = path.join(callerRoot, DOWNLOADS_DIRECTORY)+ const directory = path.join(downloadsRoot, `input-${id}`)+ const transfer: ActiveTransfer = {+ id,+ callerIdentity: input.callerIdentity,+ callerRoot,+ source: input.source,+ directory,+ sizeBytes: input.sizeBytes,+ createdAt: new Date(createdAtMs).toISOString(),+ expiresAt: input.expiresAt,+ }++ await this.serialize(async () => {+ this.assertAdmitting()+ if (this.active.size >= GFS_HOST_ACTIVE_DOWNLOADS) {+ recordGfsDownloadQuota('host', 'active_downloads')+ throw new GfsDownloadStoreError('download_busy')+ }+ if (+ (this.activeByCaller.get(input.callerIdentity) ?? 0) >=+ GFS_FILE_LIMITS.callerActiveDownloads+ ) {+ recordGfsDownloadQuota('caller', 'active_downloads')+ throw new GfsDownloadStoreError('download_busy')+ }+ if (input.retentionOwnerId !== undefined) this.assertReceiptOwner(input.retentionOwnerId)+ // Verified free capacity first: no eviction happens for an admission the+ // volume cannot hold anyway. Measured again after eviction below.+ await this.assertPhysicalCapacity(input.sizeBytes)+ await this.sweep(Date.now())+ await this.reclaimForAdmission(callerRoot, input.sizeBytes)+ const denial = this.quotaDenial(callerRoot, input.sizeBytes)+ if (denial) {+ recordGfsDownloadQuota(denial.scope, denial.reason)+ throw new GfsDownloadStoreError(+ denial.scope === 'host' ? 'host_quota_exceeded' : 'caller_quota_exceeded'+ )+ }+ await this.assertPhysicalCapacity(input.sizeBytes)++ this.addActive(transfer)+ try {+ await this.ensureDownloadsRoot(downloadsRoot)+ await fs.mkdir(directory, { mode: 0o700 })+ await this.verifyEntryDirectory(directory)+ transfer.directoryIdentity = await fs.lstat(directory, { bigint: true })+ const partial = await fs.open(+ path.join(directory, PARTIAL_FILE),+ constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,+ 0o600+ )+ await partial.close()+ } catch (error) {+ this.releaseActive(id)+ await this.removeDirectory(directory, 'incomplete_removed')+ if (error instanceof GfsDownloadStoreError) throw error+ throw new GfsDownloadStoreError('storage_write_failed')+ }+ if (input.retentionOwnerId !== undefined)+ this.pin(input.retentionOwnerId, input.callerIdentity, id)+ })++ const callerPath = receiptPath(id)+ return {+ id,+ path: callerPath,+ partialPath: `${callerPath}.partial`,+ sizeBytes: input.sizeBytes,+ expiresAt: input.expiresAt,+ }+ }++ /**+ * Publication order: sync source.partial, write meta.json (tmp + rename),+ * rename source.partial to source, sync the directory. A crash between the+ * renames leaves meta.json without source, which the sweep removes. Any+ * failure leaves the id active so the caller's fail() removes the directory.+ */+ async publish(+ id: string,+ callerIdentity: string,+ sha256: string,+ publication?: { signal?: AbortSignal; deadlineMs?: number }+ ): Promise<GfsDownloadReceipt> {+ this.assertInitialized()+ if (!SHA256_RE.test(sha256)) throw new RangeError('GFS download digest is malformed')+ const transfer = this.active.get(id)+ // Another caller's transfer or copy is answered exactly like an unknown id.+ if (!transfer || transfer.callerIdentity !== callerIdentity)+ throw new GfsDownloadStoreError('download_missing')+ const entry = await this.serialize(async () => {+ // close() may have timed out while this waited: the reservation is gone.+ if (this.closed) throw new GfsDownloadStoreError('download_busy')+ if (this.active.get(id) !== transfer) throw new GfsDownloadStoreError('download_missing')+ if (Date.parse(transfer.expiresAt) <= Date.now())+ throw new GfsDownloadStoreError('download_expired')+ this.assertPublicationOpen(publication)+ let published: Entry+ try {+ published = await this.writePublication(transfer, sha256, publication)+ } catch (error) {+ if (error instanceof GfsDownloadStoreError) throw error+ throw new GfsDownloadStoreError('storage_write_failed')+ }+ // Indexed and released in the same critical section, so no sweep can+ // observe the directory as neither active nor indexed.+ this.entries.set(id, published)+ this.releaseActive(id)+ return published+ })+ return this.receiptFor(entry, entry.source)+ }++ private async writePublication(+ transfer: ActiveTransfer,+ sha256: string,+ publication?: { signal?: AbortSignal; deadlineMs?: number }+ ): Promise<Entry> {+ const partialPath = path.join(transfer.directory, PARTIAL_FILE)+ const sourcePath = path.join(transfer.directory, SOURCE_FILE)+ const metaPath = path.join(transfer.directory, META_FILE)+ // The caller directory may have been replaced by a symlink since admission.+ await this.verifyEntryDirectory(transfer.directory)+ const partial = await openPrivateStoreObject(+ partialPath,+ 'file',+ constants.O_RDONLY,+ false+ ).catch(() => {+ throw new GfsDownloadStoreError('download_missing')+ })+ let sourceIdentity: InodeIdentity+ try {+ // A path swapped after the check above resolves to another directory:+ // the input directory must still be the one created at admission.+ await this.assertTransferDirectory(transfer)+ sourceIdentity = await partial.stat({ bigint: true })+ const info = await partial.stat()+ if (!info.isFile() || info.size !== transfer.sizeBytes)+ throw new GfsDownloadStoreError('download_missing')+ const digest = await sha256FileHandle(partial, () => this.assertPublicationOpen(publication))+ if (digest !== sha256) throw new GfsDownloadStoreError('storage_write_failed')+ const named = await fs.lstat(partialPath)+ if (named.isSymbolicLink() || named.dev !== info.dev || named.ino !== info.ino)+ throw new GfsDownloadStoreError('download_missing')+ await partial.chmod(0o600)+ await partial.sync()+ } finally {+ await partial.close()+ }++ const meta: StoredMeta = {+ schemaVersion: 1,+ id: transfer.id,+ callerIdentity: transfer.callerIdentity,+ source: transfer.source,+ sizeBytes: transfer.sizeBytes,+ sha256,+ createdAt: transfer.createdAt,+ expiresAt: transfer.expiresAt,+ }+ const temporary = path.join(transfer.directory, `${META_FILE}.tmp-${randomUUID()}`)+ let temporaryExists = false+ try {+ const handle = await fs.open(+ temporary,+ constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,+ 0o600+ )+ temporaryExists = true+ try {+ await handle.writeFile(JSON.stringify(meta), 'utf8')+ await handle.sync()+ } finally {+ await handle.close()+ }+ this.assertPublicationOpen(publication)+ await this.assertTransferDirectory(transfer)+ await fs.rename(temporary, metaPath)+ temporaryExists = false+ } finally {+ if (temporaryExists) await fs.rm(temporary, { force: true }).catch(() => undefined)+ }+ // The last cancellation point: once source exists the download is complete.+ this.assertPublicationOpen(publication)+ await this.assertTransferDirectory(transfer)+ await fs.rename(partialPath, sourcePath)+ await syncDirectory(transfer.directory)+ // Only the inode that was hashed is published, wherever the name now leads.+ if (!sameInode(await fs.lstat(sourcePath, { bigint: true }), sourceIdentity))+ throw new GfsDownloadStoreError('storage_write_failed')+ return {+ ...meta,+ directory: transfer.directory,+ sourceIdentity: { dev: sourceIdentity.dev, ino: sourceIdentity.ino },+ callerRoot: transfer.callerRoot,+ provenance: 'published',+ lastUsedMs: Date.now(),+ }+ }++ /** The transfer's input directory is still the inode created at admission. */+ private async assertTransferDirectory(transfer: ActiveTransfer): Promise<void> {+ const current = await fs.lstat(transfer.directory, { bigint: true })+ if (+ !transfer.directoryIdentity ||+ !current.isDirectory() ||+ !sameInode(current, transfer.directoryIdentity)+ )+ throw new GfsDownloadStoreError('workspace_unavailable')+ }++ /** Reuse only after the caller has freshly authorized this exact source version. */+ async reusableReceipt(+ callerIdentity: string,+ source: GfsImageSource,+ sizeBytes: number,+ bounds?: { signal?: AbortSignal; deadlineMs?: number; retentionOwnerId?: string }+ ): Promise<GfsDownloadReceipt | undefined> {+ this.assertAdmitting()+ return this.serialize(async () => {+ this.assertAdmitting()+ this.assertPublicationOpen(bounds)+ if (bounds?.retentionOwnerId !== undefined) this.assertReceiptOwner(bounds.retentionOwnerId)+ // Only copies this process published: an adopted meta.json carries an+ // identity and a digest that anyone with the Host UID could have written.+ // A miss is final; the caller downloads the file again.+ const entry = [...this.entries.values()].find(+ candidate =>+ candidate.provenance === 'published' &&+ candidate.callerIdentity === callerIdentity &&+ Date.parse(candidate.expiresAt) > Date.now() &&+ candidate.sizeBytes === sizeBytes &&+ candidate.source.drive === source.drive &&+ candidate.source.resourceId === source.resourceId &&+ candidate.source.version === source.version+ )+ if (!entry) return undefined+ let verified = false+ try {+ const handle = await this.openEntryContent(entry)+ try {+ const info = await handle.stat()+ verified =+ info.isFile() &&+ info.size === sizeBytes &&+ (info.mode & 0o777) === 0o600 &&+ (await sha256FileHandle(handle, () => this.assertPublicationOpen(bounds))) ===+ entry.sha256+ } finally {+ await handle.close()+ }+ } catch (error) {+ if (error instanceof GfsDownloadStoreError && error.code === 'publication_cancelled')+ throw error+ verified = false+ }+ if (!verified) {+ await this.removeEntry(entry, 'incomplete_removed')+ return undefined+ }+ this.assertPublicationOpen(bounds)+ entry.lastUsedMs = Date.now()+ if (bounds?.retentionOwnerId !== undefined)+ this.pin(bounds.retentionOwnerId, callerIdentity, entry.id)+ return this.receiptFor(entry, source)+ })+ }++ /**+ * Called by the producer after a transfer that was not published. The+ * directory is removed and the reservation released; a publish that failed+ * keeps its id active precisely so this call can clean it up.+ */+ async fail(id: string, callerIdentity: string): Promise<void> {+ const transfer = this.active.get(id)+ // Another caller's transfer or copy is answered exactly like an unknown id.+ if (!transfer || transfer.callerIdentity !== callerIdentity)+ throw new GfsDownloadStoreError('download_busy')+ await this.serialize(async () => {+ if (this.closed) throw new GfsDownloadStoreError('download_busy')+ // Publication may have settled while this waited.+ if (this.active.get(id) !== transfer) throw new GfsDownloadStoreError('download_busy')+ this.releaseActive(id)+ this.unpinId(id)+ if (!(await this.removeDirectory(transfer.directory, 'incomplete_removed')))+ throw new GfsDownloadStoreError('storage_write_failed')+ })+ }++ async readManagedFile(callerRelativePath: string, callerIdentity: string): Promise<Buffer> {+ const entry = this.managedEntry(callerRelativePath, callerIdentity)+ let bytes: Buffer+ try {+ const handle = await this.openEntryContent(entry)+ try {+ const info = await handle.stat()+ if (!info.isFile() || info.size !== entry.sizeBytes || (info.mode & 0o777) !== 0o600)+ throw new GfsDownloadStoreError('download_missing')+ bytes = Buffer.alloc(entry.sizeBytes)+ let offset = 0+ while (offset < entry.sizeBytes) {+ const { bytesRead } = await handle.read(bytes, offset, entry.sizeBytes - offset, offset)+ if (bytesRead <= 0) throw new GfsDownloadStoreError('download_missing')+ offset += bytesRead+ }+ const probe = Buffer.alloc(1)+ const extra = await handle.read(probe, 0, 1, entry.sizeBytes)+ if (extra.bytesRead !== 0) throw new GfsDownloadStoreError('download_missing')+ } finally {+ await handle.close()+ }+ } catch {+ throw new GfsDownloadStoreError('download_missing')+ }+ if (createHash('sha256').update(bytes).digest('hex') !== entry.sha256) {+ // A copy whose bytes no longer match its receipt is not a cache entry.+ await this.serialize(() => this.removeEntry(entry, 'incomplete_removed'))+ throw new GfsDownloadStoreError('download_missing')+ }+ return bytes+ }++ async readManagedFilePrefix(+ callerRelativePath: string,+ callerIdentity: string,+ prefixBytes = 16+ ): Promise<Buffer> {+ this.assertReadable()+ if (!Number.isSafeInteger(prefixBytes) || prefixBytes <= 0 || prefixBytes > 4096)+ throw new RangeError('GFS managed-file prefix must be between 1 and 4096 bytes')+ const entry = this.managedEntry(callerRelativePath, callerIdentity)+ try {+ const handle = await this.openEntryContent(entry)+ try {+ const info = await handle.stat()+ if (!info.isFile() || info.size !== entry.sizeBytes || (info.mode & 0o777) !== 0o600)+ throw new GfsDownloadStoreError('download_missing')+ const prefix = Buffer.alloc(prefixBytes)+ const { bytesRead } = await handle.read(prefix, 0, prefix.byteLength, 0)+ return prefix.subarray(0, bytesRead)+ } finally {+ await handle.close()+ }+ } catch {+ throw new GfsDownloadStoreError('download_missing')+ }+ }++ /**+ * Caller/task lifecycle must prove consumer settlement before invoking this.+ * Releases only the caller's own pin; an owner id another caller uses is a+ * different pin, so the call is the same no-op as for an unknown owner.+ */+ async releaseReceiptOwner(ownerId: string, callerIdentity: string): Promise<void> {+ this.assertReadable()+ await this.serialize(async () => {+ this.pins.delete(pinKey(callerIdentity, ownerId))+ })+ }++ /** Removes expired and incomplete downloads; startup runs the same sweep. */+ async cleanupExpired(now = Date.now()): Promise<GfsDownloadSweepResult> {+ this.assertReadable()+ try {+ const totals = await this.serialize(() => this.sweep(now))+ return {+ removedExpired: totals.removedExpired,+ removedIncomplete: totals.removedIncomplete,+ removeFailed: totals.removeFailed,+ }+ } catch (error) {+ recordGfsDownloadExpiry('sweep_failed')+ throw error+ }+ }++ /**+ * Stops admitting, waits for active transfers until the deadline and then+ * closes regardless. Transfers still active afterwards get download_busy+ * from publish/fail; their directories are removed by the next start.+ */+ async close(drainTimeoutMs = 5_000): Promise<void> {+ if (this.closed || this.closing) return+ if (!this.initialized) {+ this.closed = true+ return+ }+ this.closing = true+ let timer: NodeJS.Timeout | undefined+ const deadline = new Promise<void>(resolve => {+ timer = setTimeout(resolve, drainTimeoutMs)+ })+ const drained = new Promise<void>(resolve => {+ if (this.active.size === 0) resolve()+ else this.drainWaiters.push(resolve)+ })+ // Active transfers first, then whatever mutation is still in flight; both+ // bounded by the same deadline.+ await Promise.race([drained.then(() => this.mutationTail), deadline])+ clearTimeout(timer)+ if (this.active.size > 0)+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, active: this.active.size },+ 'GFS download store closed with active transfers; their directories are removed at the next start'+ )+ this.closed = true+ this.closing = false+ }++ /** Test-only: complete downloads on disk plus in-memory reservations. Read-only. */+ async debugInventory(): Promise<GfsDownloadInventory> {+ const inventory: GfsDownloadInventory = { bytes: 0, files: 0, byCaller: new Map() }+ const add = (callerRoot: string, sizeBytes: number) => {+ const key = path.basename(callerRoot)+ inventory.bytes += sizeBytes+ inventory.files += 1+ const caller = inventory.byCaller.get(key) ?? { bytes: 0, files: 0 }+ caller.bytes += sizeBytes+ caller.files += 1+ inventory.byCaller.set(key, caller)+ }+ const now = Date.now()+ for (const { id, directory } of await this.listInputDirectories()) {+ if (this.active.get(id)?.directory === directory) continue+ const state = await this.inspectDirectory(directory, id, now)+ if (state.state === 'complete') add(state.entry.callerRoot, state.entry.sizeBytes)+ }+ for (const transfer of this.active.values()) add(transfer.callerRoot, transfer.sizeBytes)+ return inventory+ }++ /**+ * Walks `users/*\/.gfs-downloads/*` plus every indexed directory. Incomplete+ * directories and expired, unpinned complete ones are removed; complete ones+ * are (re)indexed, new ones as adopted. Active transfers of this process are+ * never touched, and a published entry is never replaced: another directory+ * carrying the id of a reservation or of a published copy is removed, and an+ * id found in several directories with neither is removed everywhere (a+ * random uuid does not repeat by chance).+ */+ private async sweep(now: number): Promise<SweepTotals> {+ const totals: SweepTotals = {+ removedExpired: 0,+ removedIncomplete: 0,+ removeFailed: 0,+ retainedCompleted: 0,+ retainedBytes: 0,+ adopted: 0,+ }+ const candidates = new Map<string, string[]>()+ const addCandidate = (id: string, directory: string) => {+ const directories = candidates.get(id) ?? []+ if (!directories.includes(directory)) directories.push(directory)+ candidates.set(id, directories)+ }+ for (const { id, directory } of await this.listInputDirectories(totals))+ addCandidate(id, directory)+ for (const entry of this.entries.values()) addCandidate(entry.id, entry.directory)+ const removeIncomplete = async (directory: string) => {+ if (await this.removeDirectory(directory, 'incomplete_removed')) totals.removedIncomplete += 1+ else totals.removeFailed += 1+ }++ for (const [id, directories] of candidates) {+ const indexed = this.entries.get(id)+ const owned =+ this.active.get(id)?.directory ??+ (indexed?.provenance === 'published' ? indexed.directory : undefined)+ if (owned === undefined && directories.length > 1) {+ this.forget(id)+ for (const duplicate of directories) await removeIncomplete(duplicate)+ continue+ }+ if (owned !== undefined)+ for (const duplicate of directories)+ if (duplicate !== owned) await removeIncomplete(duplicate)+ if (this.active.has(id)) continue+ const directory = owned ?? directories[0]!+ const state =+ indexed && indexed.directory === directory+ ? await this.recheckIndexed(indexed)+ : await this.inspectDirectory(directory, id, now)+ if (state.state === 'absent') {+ this.forget(id)+ continue+ }+ if (state.state === 'incomplete') {+ this.forget(id)+ await removeIncomplete(directory)+ continue+ }+ const entry = state.entry+ if (Date.parse(entry.expiresAt) <= now && !this.isPinned(id)) {+ this.forget(id)+ if (await this.removeDirectory(directory, 'expired_removed')) totals.removedExpired += 1+ else totals.removeFailed += 1+ continue+ }+ this.entries.set(id, entry)+ totals.retainedCompleted += 1+ totals.retainedBytes += entry.sizeBytes+ if (entry.provenance === 'adopted') totals.adopted += 1+ }+ return totals+ }++ /**+ * Lists store-owned directories. A `.gfs-downloads` that is a symlink or not+ * a directory is removed without following it; names that do not match+ * `input-<uuid>` are left alone because they are not the store's.+ */+ private async listInputDirectories(+ totals?: SweepTotals+ ): Promise<Array<{ id: string; directory: string }>> {+ const found: Array<{ id: string; directory: string }> = []+ const usersRoot = path.join(this.hostRoot, 'users')+ let users: import('node:fs').Dirent[]+ try {+ const info = await fs.lstat(usersRoot)+ if (!info.isDirectory()) return found+ users = await fs.readdir(usersRoot, { withFileTypes: true })+ } catch (error) {+ if (errorCode(error) === 'ENOENT') return found+ this.reportSweepFailure(error)+ return found+ }+ for (const user of users) {+ if (!user.isDirectory()) continue+ const downloadsRoot = path.join(usersRoot, user.name, DOWNLOADS_DIRECTORY)+ let children: import('node:fs').Dirent[]+ try {+ const info = await fs.lstat(downloadsRoot)+ if (!info.isDirectory()) {+ if (totals) {+ if (await this.removeDirectory(downloadsRoot, 'incomplete_removed'))+ totals.removedIncomplete += 1+ else totals.removeFailed += 1+ }+ continue+ }+ children = await fs.readdir(downloadsRoot, { withFileTypes: true })+ } catch (error) {+ if (errorCode(error) === 'ENOENT') continue+ this.reportSweepFailure(error)+ continue+ }+ for (const child of children) {+ const match = GFS_INPUT_DIRECTORY_RE.exec(child.name)+ if (match) found.push({ id: match[1]!, directory: path.join(downloadsRoot, child.name) })+ else if (totals && TRASH_DIRECTORY_RE.test(child.name)) {+ // Left by a removal that stopped between its rename and its rm.+ if (+ await this.removeDirectory(path.join(downloadsRoot, child.name), 'incomplete_removed')+ )+ totals.removedIncomplete += 1+ else totals.removeFailed += 1+ }+ }+ }+ return found+ }++ /** Reads one directory from disk; a complete result is always `adopted`. */+ private async inspectDirectory(+ directory: string,+ id: string,+ now: number+ ): Promise<DirectoryState> {+ let info: import('node:fs').Stats+ try {+ info = await fs.lstat(directory)+ } catch (error) {+ if (errorCode(error) === 'ENOENT') return { state: 'absent' }+ return { state: 'incomplete' }+ }+ if (!info.isDirectory()) return { state: 'incomplete' }+ let raw: string+ try {+ const handle = await fs.open(+ path.join(directory, META_FILE),+ constants.O_RDONLY | constants.O_NOFOLLOW+ )+ try {+ const metaInfo = await handle.stat()+ if (!metaInfo.isFile() || metaInfo.size > META_MAX_BYTES) return { state: 'incomplete' }+ raw = await handle.readFile('utf8')+ } finally {+ await handle.close()+ }+ } catch {+ return { state: 'incomplete' }+ }+ const meta = parseMeta(raw, id, now)+ if (!meta) return { state: 'incomplete' }+ try {+ const source = await fs.lstat(path.join(directory, SOURCE_FILE))+ if (!source.isFile() || source.size !== meta.sizeBytes) return { state: 'incomplete' }+ } catch {+ return { state: 'incomplete' }+ }+ return {+ state: 'complete',+ entry: {+ ...meta,+ directory,+ callerRoot: callerRootOf(directory),+ provenance: 'adopted',+ lastUsedMs: Date.parse(meta.createdAt),+ },+ }+ }++ /** An indexed entry is re-checked with one lstat; meta.json is not re-read. */+ private async recheckIndexed(entry: Entry): Promise<DirectoryState> {+ try {+ const source = await fs.lstat(path.join(entry.directory, SOURCE_FILE))+ if (source.isFile() && source.size === entry.sizeBytes) return { state: 'complete', entry }+ } catch (error) {+ if (errorCode(error) === 'ENOENT') {+ try {+ await fs.lstat(entry.directory)+ } catch (directoryError) {+ if (errorCode(directoryError) === 'ENOENT') return { state: 'absent' }+ }+ }+ }+ return { state: 'incomplete' }+ }++ /**+ * Index lookup for a managed read. Only a copy this process published for+ * this caller is served. An adopted, unknown or another caller's id is the+ * same download_missing, decided from the index alone: the disk is not+ * probed, because nothing on disk can prove who downloaded a copy, and a+ * foreign copy must cost no more work than a missing one.+ */+ private managedEntry(callerRelativePath: string, callerIdentity: string): Entry {+ this.assertReadable()+ const match = RECEIPT_PATH_RE.exec(callerRelativePath)+ if (!match) throw new GfsDownloadStoreError('download_missing')+ const entry = this.entries.get(match[1]!)+ if (!entry || entry.provenance !== 'published' || entry.callerIdentity !== callerIdentity)+ throw new GfsDownloadStoreError('download_missing')+ if (Date.parse(entry.expiresAt) <= Date.now())+ throw new GfsDownloadStoreError('download_expired')+ entry.lastUsedMs = Date.now()+ return entry+ }++ private async openEntryContent(entry: Entry): Promise<fs.FileHandle> {+ await this.verifyEntryDirectory(entry.directory)+ const handle = await openPrivateStoreObject(+ path.join(entry.directory, SOURCE_FILE),+ 'file',+ constants.O_RDONLY,+ false+ )+ try {+ // The descriptor, not the path, decides: a name swapped after the+ // directory check opens another inode and is refused.+ if (+ !entry.sourceIdentity ||+ !sameInode(await handle.stat({ bigint: true }), entry.sourceIdentity)+ )+ throw new GfsDownloadStoreError('download_missing')+ const info = await handle.stat()+ if (info.size !== entry.sizeBytes) throw new GfsDownloadStoreError('download_missing')+ if ((info.mode & 0o7777) !== 0o600) {+ // Restore the private mode only for content that still matches.+ if ((await sha256FileHandle(handle)) !== entry.sha256)+ throw new GfsDownloadStoreError('download_missing')+ await handle.chmod(0o600)+ }+ return handle+ } catch (error) {+ await handle.close()+ throw error+ }+ }++ private async verifyEntryDirectory(directory: string): Promise<void> {+ if (!isWithinDirectory(directory, this.hostRoot))+ throw new GfsDownloadStoreError('workspace_unavailable')+ const callerRoot = path.dirname(path.dirname(directory))+ if ((await fs.realpath(callerRoot)) !== callerRoot)+ throw new GfsDownloadStoreError('workspace_unavailable')+ await verifyPrivateStoreDirectory(path.dirname(directory))+ await verifyPrivateStoreDirectory(directory)+ }++ /**+ * A `.gfs-downloads` that is a symlink, not a directory, or not private to+ * this process is removed without following it and recreated, so a shell+ * command can cost a caller its cached copies but never block its downloads.+ */+ private async ensureDownloadsRoot(downloadsRoot: string): Promise<void> {+ try {+ await fs.mkdir(downloadsRoot, { mode: 0o700 })+ return+ } catch (error) {+ if (errorCode(error) !== 'EEXIST') throw error+ }+ try {+ await verifyPrivateStoreDirectory(downloadsRoot)+ return+ } catch {+ // Not a private directory: replaced below.+ }+ for (const id of this.idsUnder(downloadsRoot)) this.forget(id)+ await this.removeVerified(downloadsRoot)+ await fs.mkdir(downloadsRoot, { mode: 0o700 })+ await verifyPrivateStoreDirectory(downloadsRoot)+ }++ private idsUnder(downloadsRoot: string): string[] {+ return [...this.entries.values()]+ .filter(entry => path.dirname(entry.directory) === downloadsRoot)+ .map(entry => entry.id)+ }++ private async assertPhysicalCapacity(sizeBytes: number): Promise<void> {+ const space = await fs.statfs(this.hostRoot, { bigint: true })+ if (space.bsize <= 0n || space.bavail < 0n) {+ recordGfsDownloadQuota('host', 'free_space')+ throw new GfsDownloadStoreError('host_quota_exceeded')+ }+ const reserve = (bytes: number): bigint => {+ const amount = BigInt(bytes)+ return ((amount + space.bsize - 1n) / space.bsize) * space.bsize+ }+ let required = reserve(sizeBytes) + FREE_SPACE_RESERVE_BYTES+ // Use the full outstanding reservation of every active transfer rather+ // than crediting sparse, shared or concurrently written allocation.+ for (const transfer of this.active.values()) required += reserve(transfer.sizeBytes)+ if (space.bavail * space.bsize < required) {+ recordGfsDownloadQuota('host', 'free_space')+ throw new GfsDownloadStoreError('host_quota_exceeded')+ }+ }++ /** Caller usage is keyed by the caller directory that holds each copy, never by meta.json. */+ private usage(excluded: ReadonlySet<string> = new Set()) {+ const usage = {+ bytes: 0,+ files: 0,+ callerBytes: new Map<string, number>(),+ callerFiles: new Map<string, number>(),+ }+ const add = (callerRoot: string, sizeBytes: number) => {+ usage.bytes += sizeBytes+ usage.files += 1+ usage.callerBytes.set(callerRoot, (usage.callerBytes.get(callerRoot) ?? 0) + sizeBytes)+ usage.callerFiles.set(callerRoot, (usage.callerFiles.get(callerRoot) ?? 0) + 1)+ }+ for (const entry of this.entries.values())+ if (!excluded.has(entry.id)) add(entry.callerRoot, entry.sizeBytes)+ for (const transfer of this.active.values()) add(transfer.callerRoot, transfer.sizeBytes)+ return usage+ }++ private quotaDenial(+ callerRoot: string,+ sizeBytes: number,+ excluded?: ReadonlySet<string>+ ): { scope: 'host' | 'caller'; reason: 'storage_bytes' | 'retained_files' } | undefined {+ const usage = this.usage(excluded)+ // Prove caller admission before considering eviction of another caller.+ if ((usage.callerBytes.get(callerRoot) ?? 0) + sizeBytes > GFS_FILE_LIMITS.callerStorageBytes)+ return { scope: 'caller', reason: 'storage_bytes' }+ if ((usage.callerFiles.get(callerRoot) ?? 0) >= GFS_FILE_LIMITS.callerRetainedFiles)+ return { scope: 'caller', reason: 'retained_files' }+ if (usage.bytes + sizeBytes > GFS_FILE_LIMITS.storageBytes)+ return { scope: 'host', reason: 'storage_bytes' }+ if (usage.files >= GFS_HOST_RETAINED_FILES) return { scope: 'host', reason: 'retained_files' }+ return undefined+ }++ /**+ * Completed, unpinned copies are a cache. The whole eviction plan is computed+ * before anything is deleted: adopted copies first, so files planted in one+ * directory can never cost another caller a copy this process published,+ * then published copies least recently used first. A caller denial evicts+ * only copies in that caller's directory. With no feasible plan nothing is+ * deleted and the admission fails with the quota code; nothing is retained,+ * so the next admission plans again. Candidates are not hashed: a corrupt+ * copy is as good an eviction candidate as a sound one.+ */+ private async reclaimForAdmission(callerRoot: string, sizeBytes: number): Promise<void> {+ if (!this.quotaDenial(callerRoot, sizeBytes)) return+ const candidates = [...this.entries.values()]+ .filter(entry => !this.isPinned(entry.id))+ .sort(evictionOrder)+ const plan = new Set<string>()+ for (;;) {+ const denial = this.quotaDenial(callerRoot, sizeBytes, plan)+ if (!denial) break+ const next = candidates.find(+ entry => !plan.has(entry.id) && (denial.scope === 'host' || entry.callerRoot === callerRoot)+ )+ if (!next) return+ plan.add(next.id)+ }+ for (const id of plan) {+ const entry = this.entries.get(id)+ if (entry) await this.removeEntry(entry, 'expired_removed', false)+ }+ }++ /** Removes an indexed entry's directory and forgets it. */+ private async removeEntry(+ entry: Entry,+ outcome: 'incomplete_removed' | 'expired_removed',+ count = true+ ): Promise<void> {+ this.forget(entry.id)+ await this.removeDirectory(entry.directory, count ? outcome : undefined)+ }++ /**+ * `fs.rm` does not follow the final component, but it resolves every parent+ * component, so a caller directory or `.gfs-downloads` replaced by a symlink+ * after indexing would point the removal outside the Host root. The parent+ * must therefore be its own real path inside the Host root, and the+ * directory must be gone afterwards. A refusal or a failure is logged with+ * its code and counted; the next sweep retries what is still listed.+ */+ private async removeDirectory(+ directory: string,+ outcome: 'incomplete_removed' | 'expired_removed' | undefined+ ): Promise<boolean> {+ try {+ await this.removeVerified(directory)+ } catch (error) {+ recordGfsDownloadExpiry('remove_failed')+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: errorCode(error) },+ 'GFS download store could not remove a download directory; the next sweep retries it'+ )+ return false+ }+ if (outcome) recordGfsDownloadExpiry(outcome)+ return true+ }++ /**+ * Node has no unlinkat/renameat, so the window between a path check and a+ * path-based removal cannot be closed, only narrowed. The entry is first+ * renamed to a store-private name inside its verified parent; the parent is+ * then verified again, and only that private name is removed. When the+ * parent moved in between, the rename is undone and the removal refused,+ * so whatever the swapped path led to keeps its name and content.+ */+ private async removeVerified(directory: string): Promise<void> {+ await this.assertRemovable(directory)+ const trash = path.join(path.dirname(directory), `${TRASH_PREFIX}${randomUUID()}`)+ try {+ await fs.rename(directory, trash)+ } catch (error) {+ if (errorCode(error) !== 'ENOENT') throw error+ await assertAbsent(directory)+ return+ }+ try {+ await this.assertRemovable(trash)+ } catch (error) {+ await fs.rename(trash, directory)+ throw error+ }+ await fs.rm(trash, { recursive: true, force: true })+ await assertAbsent(trash)+ }++ /** The parent of a directory about to be removed is real and inside the Host root. */+ private async assertRemovable(directory: string): Promise<void> {+ const parent = path.dirname(directory)+ if (!isWithinDirectory(parent, this.hostRoot))+ throw new GfsDownloadStoreError('workspace_unavailable')+ let real: string+ try {+ real = await fs.realpath(parent)+ } catch (error) {+ // No parent, nothing to remove: `fs.rm` with `force` is a no-op.+ if (errorCode(error) === 'ENOENT') return+ throw error+ }+ if (real !== parent) throw new GfsDownloadStoreError('workspace_unavailable')+ }++ private reportSweepFailure(error: unknown): void {+ recordGfsDownloadExpiry('sweep_failed')+ logger.warn(+ { component: GFS_DOWNLOAD_STORE_LOG_COMPONENT, code: errorCode(error) },+ 'GFS download store could not list a download directory; the next sweep retries it'+ )+ }++ private receiptFor(entry: Entry, source: GfsImageSource): GfsDownloadReceipt {+ return {+ id: entry.id,+ source,+ path: receiptPath(entry.id),+ sizeBytes: entry.sizeBytes,+ sha256: entry.sha256,+ expiresAt: entry.expiresAt,+ }+ }++ /** A malformed owner id is refused; pins are per caller, so no id is taken. */+ private assertReceiptOwner(ownerId: string): void {+ if (!validReceiptOwnerId(ownerId)) throw new GfsDownloadStoreError('caller_mismatch')+ }++ private pin(ownerId: string, callerIdentity: string, id: string): void {+ const key = pinKey(callerIdentity, ownerId)+ const ids = this.pins.get(key) ?? new Set<string>()+ ids.add(id)+ this.pins.set(key, ids)+ }++ private isPinned(id: string): boolean {+ for (const ids of this.pins.values()) if (ids.has(id)) return true+ return false+ }++ private unpinId(id: string): void {+ for (const [key, ids] of this.pins) {+ ids.delete(id)+ if (ids.size === 0) this.pins.delete(key)+ }+ }++ private forget(id: string): void {+ this.entries.delete(id)+ this.unpinId(id)+ }++ private addActive(transfer: ActiveTransfer): void {+ this.active.set(transfer.id, transfer)+ this.activeByCaller.set(+ transfer.callerIdentity,+ (this.activeByCaller.get(transfer.callerIdentity) ?? 0) + 1+ )+ }++ private releaseActive(id: string): void {+ const transfer = this.active.get(id)+ if (!transfer) return+ this.active.delete(id)+ const current = this.activeByCaller.get(transfer.callerIdentity) ?? 0+ if (current <= 1) this.activeByCaller.delete(transfer.callerIdentity)+ else this.activeByCaller.set(transfer.callerIdentity, current - 1)+ if (this.active.size === 0) {+ const waiters = this.drainWaiters+ this.drainWaiters = []+ for (const resolve of waiters) resolve()+ }+ }++ private async validateCallerRoot(callerRoot: string): Promise<string> {+ const lexical = path.resolve(callerRoot)+ const expected = isWithinDirectory(lexical, this.hostRoot)+ ? lexical+ : isWithinDirectory(lexical, this.requestedHostRoot)+ ? path.join(this.hostRoot, path.relative(this.requestedHostRoot, lexical))+ : undefined+ if (expected === undefined) throw new GfsDownloadStoreError('workspace_unavailable')+ const real = await fs.realpath(lexical).catch(() => {+ throw new GfsDownloadStoreError('workspace_unavailable')+ })+ if (real !== expected || !isWithinDirectory(real, await fs.realpath(this.hostRoot)))+ throw new GfsDownloadStoreError('workspace_unavailable')+ const info = await fs.lstat(real)+ if (!info.isDirectory() || info.isSymbolicLink())+ throw new GfsDownloadStoreError('workspace_unavailable')+ return real+ }++ private async serialize<T>(operation: () => Promise<T>): Promise<T> {+ const previous = this.mutationTail+ let unlock!: () => void+ this.mutationTail = new Promise<void>(resolve => {+ unlock = resolve+ })+ await previous+ try {+ return await operation()+ } finally {+ unlock()+ }+ }++ private assertPublicationOpen(publication?: { signal?: AbortSignal; deadlineMs?: number }): void {+ if (+ publication?.signal?.aborted ||+ (publication?.deadlineMs !== undefined && Date.now() >= publication.deadlineMs)+ )+ throw new GfsDownloadStoreError('publication_cancelled')+ }++ /** Not initialized: workspace_unavailable. Closed: download_busy. */+ private assertInitialized(): void {+ if (this.closed) throw new GfsDownloadStoreError('download_busy')+ if (!this.initialized) throw new GfsDownloadStoreError('workspace_unavailable')+ }++ private assertReadable(): void {+ this.assertInitialized()+ }++ /** Admission also stops while close() drains. */+ private assertAdmitting(): void {+ this.assertInitialized()+ if (this.closing) throw new GfsDownloadStoreError('download_busy')+ }+}diff --git a/mcp-host/src/internalTools/gfsDownloadStoreProvenance.test.ts b/mcp-host/src/internalTools/gfsDownloadStoreProvenance.test.tsnew file mode 100644index 000000000..ddb88f844--- /dev/null+++ b/mcp-host/src/internalTools/gfsDownloadStoreProvenance.test.ts@@ -0,0 +1,423 @@+/**+ * Provenance boundary of the GFS download store. A shell command runs with the+ * Host UID and can write anything under `users/`, so a copy found on disk+ * proves nothing about who downloaded it or what it hashes to. These tests+ * plant such copies and check that they are counted and swept but never+ * reused, read or allowed to displace a copy this process published.+ */+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import * as crypto from 'node:crypto'+import { randomUUID } from 'node:crypto'+import * as syncFs from 'node:fs'+import * as fs from 'node:fs/promises'+import { tmpdir } from 'node:os'+import * as path from 'node:path'+import {+ callerDirectory,+ completedCopy,+ digestOf,+ downloadDirectory,+ exists,+ expiryCount,+ metaFor,+ plantDownload,+ quotaCount,+ sourceFor,+ startTransfer,+ withEnvironment,+} from '../__tests__/fixtures/gfsStoreTestKit'+import { logger } from '../logger'+import { GfsDownloadStoreFs } from './gfsDownloadStoreFs'++const { hashBoundary } = vi.hoisted(() => ({ hashBoundary: vi.fn() }))+// Pass-through: the store hashes through createHash; the kit uses crypto.hash.+vi.mock('node:crypto', async original => ({+ ...(await original<typeof crypto>()),+ createHash: hashBoundary,+}))++const A = 'caller-a'+const B = 'caller-b'+const C = 'caller-c'++let nativeCrypto: typeof crypto+let hostRoot: string+let rootA: string+let rootB: string+let store: GfsDownloadStoreFs+const extraStores: GfsDownloadStoreFs[] = []+const outsideRoots: string[] = []++function receiptPathOf(id: string): string {+ return `.gfs-downloads/input-${id}/source`+}++/** A complete, well-formed copy written by a shell command, not by the store. */+function plantComplete(+ root: string,+ identity: string,+ index: number,+ bytes: Buffer,+ options: { id?: string } = {}+) {+ const id = options.id ?? randomUUID()+ return plantDownload(root, {+ id,+ bytes,+ meta: metaFor(id, bytes, Date.now(), { callerIdentity: identity, source: sourceFor(index) }),+ })+}++async function openStore(): Promise<GfsDownloadStoreFs> {+ const opened = new GfsDownloadStoreFs(hostRoot)+ extraStores.push(opened)+ await opened.initialize()+ return opened+}++function outsideDirectory(): string {+ const outside = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-outside-'))+ outsideRoots.push(outside)+ return outside+}++async function expectNotServed(target: GfsDownloadStoreFs, id: string, identity: string) {+ await expect(target.readManagedFile(receiptPathOf(id), identity)).rejects.toMatchObject({+ code: 'download_missing',+ })+ await expect(target.readManagedFilePrefix(receiptPathOf(id), identity)).rejects.toMatchObject({+ code: 'download_missing',+ })+}++function adoptedLogged(info: ReturnType<typeof vi.spyOn>, adopted: number) {+ expect(info).toHaveBeenCalledWith(+ expect.objectContaining({ component: 'GfsDownloadStore', adopted }),+ 'GFS download store initialized'+ )+}++beforeEach(async () => {+ nativeCrypto = await vi.importActual<typeof crypto>('node:crypto')+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))+ hashBoundary.mockReset()+ hashBoundary.mockImplementation((algorithm: string) => nativeCrypto.createHash(algorithm))+ hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-provenance-'))+ rootA = callerDirectory(hostRoot, A)+ rootB = callerDirectory(hostRoot, B)+ store = new GfsDownloadStoreFs(hostRoot)+ await store.initialize()+})++afterEach(async () => {+ vi.restoreAllMocks()+ vi.useRealTimers()+ for (const opened of extraStores.splice(0)) await opened.close(0)+ await store.close(0)+ await fs.rm(hostRoot, { recursive: true, force: true })+ for (const outside of outsideRoots.splice(0))+ await fs.rm(outside, { recursive: true, force: true })+})++describe('GFS download store provenance: planted copies', () => {+ it("T-S1: a planted entry in the caller's directory is never reused", async () => {+ const bytes = Buffer.alloc(32, 0x41)+ const planted = plantComplete(rootB, B, 5, bytes)+ const info = vi.spyOn(logger, 'info')+ const restarted = await openStore()++ // Witnesses: the restarted store indexed the copy and counts it under B.+ adoptedLogged(info, 1)+ expect((await restarted.debugInventory()).byCaller.get(B)).toEqual({ files: 1, bytes: 32 })+ hashBoundary.mockClear()++ await expect(restarted.reusableReceipt(B, sourceFor(5), 32)).resolves.toBeUndefined()+ expect(hashBoundary).not.toHaveBeenCalled()+ const transfer = await restarted.createTransfer({+ callerIdentity: B,+ callerWorkspacePath: rootB,+ source: sourceFor(5),+ sizeBytes: 32,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ })+ expect(transfer.id).not.toBe(planted.id)+ expect(exists(planted.sourcePath)).toBe(true)+ })++ it('T-S2: a planted entry is never read through the managed path', async () => {+ const own = await completedCopy(store, rootB, B, 6, 16)+ const bytes = Buffer.alloc(24, 0x42)+ const planted = plantComplete(rootB, B, 7, bytes)++ // Not yet indexed: the managed path must not look at the disk.+ await expectNotServed(store, planted.id, B)+ // Control: the same read path serves B's published copy.+ await expect(store.readManagedFile(own.receipt.path, B)).resolves.toEqual(own.bytes)++ const info = vi.spyOn(logger, 'info')+ const restarted = await openStore()+ adoptedLogged(info, 2)+ expect((await restarted.debugInventory()).byCaller.get(B)).toEqual({ files: 2, bytes: 40 })+ await expectNotServed(restarted, planted.id, B)+ expect(exists(planted.sourcePath)).toBe(true)+ })++ it('T-S3: a planted directory with a published id is removed and the published copy is untouched', async () => {+ const published = await completedCopy(store, rootB, B, 8, 32)+ const forged = Buffer.alloc(32, 0x5a)+ const planted = plantComplete(rootA, B, 8, forged, { id: published.receipt.id })++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(exists(planted.directory)).toBe(false)+ const reused = await store.reusableReceipt(B, sourceFor(8), 32)+ expect(reused).toMatchObject({ id: published.receipt.id, sha256: digestOf(published.bytes) })+ await expect(store.readManagedFile(published.receipt.path, B)).resolves.toEqual(published.bytes)+ })++ it('T-S4: a planted entry is charged to the directory that contains it', async () => {+ const bytes = Buffer.alloc(10, 0x43)+ const planted = plantComplete(rootB, C, 9, bytes)+ const limited = await withEnvironment(+ {+ MCP_HOST_GFS_MAX_FILE_BYTES: '10',+ MCP_HOST_GFS_DOWNLOAD_STORAGE_BYTES: '30',+ MCP_HOST_GFS_CALLER_DOWNLOAD_STORAGE_BYTES: '15',+ },+ async () => {+ vi.resetModules()+ const { GfsDownloadStoreFs: Store } = await import('./gfsDownloadStoreFs')+ const opened = new Store(hostRoot)+ extraStores.push(opened as unknown as GfsDownloadStoreFs)+ await opened.initialize()+ return opened+ }+ )+ const inventory = await limited.debugInventory()+ expect(inventory.byCaller.get(B)).toEqual({ files: 1, bytes: 10 })+ expect(inventory.byCaller.has(C)).toBe(false)+ const callerDenied = await quotaCount('caller', 'storage_bytes')++ // C's directory holds nothing, so C's full budget is free and nothing is evicted.+ const rootC = callerDirectory(hostRoot, C)+ await startTransfer(limited, rootC, C, 10, 10)+ expect(exists(planted.directory)).toBe(true)++ // B's directory holds the plant: B's admission evicts it to fit 10 + 10 into 15.+ await startTransfer(limited, rootB, B, 11, 10)+ expect(exists(planted.directory)).toBe(false)+ expect(await quotaCount('caller', 'storage_bytes')).toBe(callerDenied)+ })++ it('T-R1: a restart keeps the copy for accounting and cleanup but does not reuse it', async () => {+ const published = await completedCopy(store, rootB, B, 12, 20)+ await store.close(0)+ const info = vi.spyOn(logger, 'info')++ const restarted = await openStore()++ adoptedLogged(info, 1)+ expect((await restarted.debugInventory()).byCaller.get(B)).toEqual({ files: 1, bytes: 20 })+ await expect(restarted.reusableReceipt(B, sourceFor(12), 20)).resolves.toBeUndefined()+ await expectNotServed(restarted, published.receipt.id, B)+ const again = await startTransfer(restarted, rootB, B, 12, 20)+ expect(again.transfer.id).not.toBe(published.receipt.id)+ const directories = syncFs.readdirSync(path.join(rootB, '.gfs-downloads'))+ expect(directories.filter(name => name.startsWith('input-'))).toHaveLength(2)+ })++ it("P7: a planted entry with the caller's own identity is neither reused nor read", async () => {+ const bytes = Buffer.alloc(18, 0x44)+ const planted = plantComplete(rootA, A, 13, bytes)+ const info = vi.spyOn(logger, 'info')+ const restarted = await openStore()++ adoptedLogged(info, 1)+ expect((await restarted.debugInventory()).byCaller.get(A)).toEqual({ files: 1, bytes: 18 })+ await expect(restarted.reusableReceipt(A, sourceFor(13), 18)).resolves.toBeUndefined()+ await expectNotServed(restarted, planted.id, A)+ expect(exists(planted.sourcePath)).toBe(true)+ })++ it('P8: a meta identity that differs from its directory grants nothing to either caller', async () => {+ const bytes = Buffer.alloc(12, 0x45)+ const planted = plantComplete(rootB, C, 14, bytes)+ const info = vi.spyOn(logger, 'info')+ const restarted = await openStore()++ adoptedLogged(info, 1)+ expect((await restarted.debugInventory()).byCaller.get(B)).toEqual({ files: 1, bytes: 12 })+ await expectNotServed(restarted, planted.id, C)+ await expectNotServed(restarted, planted.id, B)+ await expect(restarted.reusableReceipt(C, sourceFor(14), 12)).resolves.toBeUndefined()+ expect(exists(planted.sourcePath)).toBe(true)+ })++ it('P9: duplicate ids without a published entry are all removed', async () => {+ const id = randomUUID()+ const first = plantComplete(rootA, A, 15, Buffer.alloc(8, 1), { id })+ const second = plantComplete(rootB, B, 15, Buffer.alloc(8, 2), { id })+ const unrelated = plantComplete(rootA, A, 16, Buffer.alloc(8, 3))++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 2 })++ expect(exists(first.directory)).toBe(false)+ expect(exists(second.directory)).toBe(false)+ // Witness: the sweep kept the unrelated complete plant.+ expect(exists(unrelated.directory)).toBe(true)+ expect((await store.debugInventory()).files).toBe(1)+ })++ it('P10: a planted directory with an active id is removed and the transfer still publishes', async () => {+ const { transfer, bytes } = await startTransfer(store, rootA, A, 17, 16)+ const planted = plantComplete(rootB, A, 17, Buffer.alloc(16, 0x46), { id: transfer.id })++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(exists(planted.directory)).toBe(false)+ const receipt = await store.publish(transfer.id, A, digestOf(bytes))+ await expect(store.readManagedFile(receipt.path, A)).resolves.toEqual(bytes)+ })++ it.each(['input directory', 'meta.json', 'source'] as const)('P11: symlink as %s', async kind => {+ const outside = outsideDirectory()+ const bytes = Buffer.alloc(20, 0x47)+ const id = randomUUID()+ const meta = metaFor(id, bytes, Date.now(), { callerIdentity: B, source: sourceFor(18) })+ const target = plantDownload(outside, { id, bytes, meta })+ const sibling = plantComplete(rootB, B, 19, Buffer.alloc(6, 0x48))+ let link: string+ if (kind === 'input directory') {+ link = downloadDirectory(rootB, id)+ syncFs.symlinkSync(target.directory, link)+ } else if (kind === 'meta.json') {+ const planted = plantDownload(rootB, { id, bytes, meta: 'omit' })+ link = path.join(planted.directory, 'meta.json')+ syncFs.symlinkSync(path.join(target.directory, 'meta.json'), link)+ } else {+ const planted = plantDownload(rootB, { id, bytes, meta, source: 'omit' })+ link = planted.sourcePath+ syncFs.symlinkSync(target.sourcePath, link)+ }++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(exists(link)).toBe(false)+ expect(exists(downloadDirectory(rootB, id))).toBe(false)+ expect(syncFs.readFileSync(target.sourcePath)).toEqual(bytes)+ expect(exists(path.join(target.directory, 'meta.json'))).toBe(true)+ // Witness: the sweep retained the real sibling copy.+ expect(exists(sibling.directory)).toBe(true)+ expect((await store.debugInventory()).files).toBe(1)+ })++ it('P15: a .gfs-downloads that is a file is removed and the caller can still download', async () => {+ const downloads = path.join(rootB, '.gfs-downloads')+ syncFs.writeFileSync(downloads, 'not a directory', { mode: 0o600 })++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 1 })++ expect(exists(downloads)).toBe(false)+ const copy = await completedCopy(store, rootB, B, 20, 9)+ await expect(store.readManagedFile(copy.receipt.path, B)).resolves.toEqual(copy.bytes)+ })++ it('P16: a symlinked caller directory is neither listed nor accepted as a caller root', async () => {+ const outside = outsideDirectory()+ const outsideCaller = path.join(outside, 'caller-x')+ syncFs.mkdirSync(outsideCaller, { mode: 0o700 })+ const hidden = plantComplete(outsideCaller, 'caller-x', 21, Buffer.alloc(10, 0x49))+ const linkedRoot = path.join(hostRoot, 'users', 'caller-x')+ syncFs.symlinkSync(outsideCaller, linkedRoot)+ const real = plantComplete(rootB, B, 22, Buffer.alloc(7, 0x4a))++ await expect(store.cleanupExpired()).resolves.toMatchObject({ removedIncomplete: 0 })+ const inventory = await store.debugInventory()++ // Witness: the real copy is listed and counted.+ expect(inventory.files).toBe(1)+ expect(inventory.byCaller.get(B)).toEqual({ files: 1, bytes: 7 })+ expect(inventory.byCaller.has('caller-x')).toBe(false)+ expect(exists(hidden.sourcePath)).toBe(true)+ expect(exists(real.sourcePath)).toBe(true)+ await expect(startTransfer(store, linkedRoot, 'caller-x', 23, 4)).rejects.toMatchObject({+ code: 'workspace_unavailable',+ })+ })++ it('P17: a hardlinked published source is not reused and is downloaded again', async () => {+ const published = await completedCopy(store, rootB, B, 24, 32)+ const directory = downloadDirectory(rootB, published.receipt.id)+ const link = path.join(outsideDirectory(), 'linked-source')+ syncFs.linkSync(path.join(directory, 'source'), link)+ const altered = Buffer.alloc(32, 0xee)+ syncFs.writeFileSync(link, altered)++ await expect(store.reusableReceipt(B, sourceFor(24), 32)).resolves.toBeUndefined()++ // Witness: the reuse path opened the copy and removed it.+ expect(exists(directory)).toBe(false)+ expect(syncFs.readFileSync(link)).toEqual(altered)+ const again = await completedCopy(store, rootB, B, 24, 32)+ expect(again.receipt.id).not.toBe(published.receipt.id)+ })++ it.each(['expiry sweep', 'failed reuse'] as const)(+ 'P18: a caller directory replaced by a symlink after publish is not followed by the %s',+ async trigger => {+ const published = await completedCopy(store, rootB, B, 25, 16)+ const outside = outsideDirectory()+ const decoy = plantDownload(outside, {+ id: published.receipt.id,+ bytes: published.bytes,+ meta: metaFor(published.receipt.id, published.bytes, Date.now(), {+ callerIdentity: B,+ source: sourceFor(25),+ }),+ })+ syncFs.renameSync(rootB, `${rootB}-moved`)+ syncFs.symlinkSync(outside, rootB)+ const failedBefore = await expiryCount('remove_failed')+ const warn = vi.spyOn(logger, 'warn')++ if (trigger === 'expiry sweep') {+ await expect(+ store.cleanupExpired(Date.parse(published.receipt.expiresAt))+ ).resolves.toMatchObject({ removedExpired: 0, removeFailed: 1 })+ } else {+ await expect(store.reusableReceipt(B, sourceFor(25), 16)).resolves.toBeUndefined()+ }++ // Witness: the removal was reached and refused on containment.+ expect(await expiryCount('remove_failed')).toBe(failedBefore + 1)+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({ component: 'GfsDownloadStore', code: 'workspace_unavailable' }),+ 'GFS download store could not remove a download directory; the next sweep retries it'+ )+ expect(syncFs.readFileSync(decoy.sourcePath)).toEqual(published.bytes)+ expect(exists(path.join(decoy.directory, 'meta.json'))).toBe(true)+ }+ )++ it('P19: a caller directory replaced by a symlink before publish is refused and nothing is written outside', async () => {+ const { transfer, bytes } = await startTransfer(store, rootB, B, 26, 12)+ const outside = outsideDirectory()+ const decoy = downloadDirectory(outside, transfer.id)+ syncFs.mkdirSync(decoy, { recursive: true, mode: 0o700 })+ syncFs.chmodSync(path.dirname(decoy), 0o700)+ syncFs.writeFileSync(path.join(decoy, 'source.partial'), bytes, { mode: 0o600 })+ syncFs.renameSync(rootB, `${rootB}-moved`)+ syncFs.symlinkSync(outside, rootB)++ await expect(store.publish(transfer.id, B, digestOf(bytes))).rejects.toMatchObject({+ code: 'workspace_unavailable',+ })++ // Witness: the partial the decoy offered is still there, unpublished.+ expect(syncFs.readFileSync(path.join(decoy, 'source.partial'))).toEqual(bytes)+ expect(exists(path.join(decoy, 'source'))).toBe(false)+ expect(exists(path.join(decoy, 'meta.json'))).toBe(false)+ await expectNotServed(store, transfer.id, B)+ })+})