Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
39.5 KB
Raw Blame
/**
 * Removal-charge accounting of the GFS download store.
 *
 * Whenever a removal leaves bytes on disk (in place, or under a `.trash-` name), those bytes stay
 * charged against the Host budget until the directory is removed or confirmed absent. Each test
 * runs on a coherent simulated volume (1000 MiB at 1%, so a 10 MiB budget) whose free space is the
 * source bytes actually present, injects one filesystem fault, asserts the refusal, then clears the
 * fault and proves the store recovers with a fresh publish and read.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import {
  callerDirectory,
  completedCopy,
  digestOf,
  downloadDirectory,
  expiryCount,
  plantDownload,
  quotaCount,
  sourceFor,
  startTransfer,
  withEnvironment,
} from '../__tests__/fixtures/gfsStoreTestKit'
import type { GfsDownloadStore } from './gfsDownloadStore'

const boundaries = vi.hoisted(() => ({
  open: vi.fn(),
  statfs: vi.fn(),
  rm: vi.fn(),
  rename: vi.fn(),
  realpath: vi.fn(),
  lstat: vi.fn(),
  readdir: vi.fn(),
}))
vi.mock('node:fs/promises', async original => ({
  ...(await original<typeof fs>()),
  ...boundaries,
}))

const MIB = 1024 * 1024
const HOUR = 60 * 60_000
const TOTAL = 1000n * BigInt(MIB)
let nativeFs: typeof fs
let hostRoot: string
let roots: Record<string, string>
const stores: GfsDownloadStore[] = []
const releases: Array<() => void> = []
const eio = () => Object.assign(new Error('EIO: injected fault'), { code: 'EIO' })

function storedBytes(directory: string): number {
  let sum = 0
  for (const child of syncFs.readdirSync(directory, { withFileTypes: true })) {
    const target = path.join(directory, child.name)
    if (child.isDirectory()) sum += storedBytes(target)
    else if (child.isFile() && ['source', 'source.partial'].includes(child.name))
      sum += syncFs.statSync(target).size
  }
  return sum
}

// Every regular file, `meta.json` included: what a chargeless measurement counts.
function regularBytes(directory: string): number {
  let sum = 0
  for (const child of syncFs.readdirSync(directory, { withFileTypes: true })) {
    const target = path.join(directory, child.name)
    if (child.isDirectory()) sum += regularBytes(target)
    else if (child.isFile()) sum += syncFs.statSync(target).size
  }
  return sum
}

function view(store: GfsDownloadStore) {
  const state = store as unknown as {
    entries: Map<string, { sizeBytes: number }>
    active: Map<string, { sizeBytes: number }>
    held: Map<string, { sizeBytes: number }>
    pins: Map<string, Set<string>>
  }
  const sum = (map: Map<string, { sizeBytes: number }>) =>
    [...map.values()].reduce((n, item) => n + item.sizeBytes, 0)
  return {
    state,
    entries: sum(state.entries),
    active: sum(state.active),
    held: sum(state.held),
    charged: sum(state.entries) + sum(state.active) + sum(state.held),
  }
}

function restoreBoundary(name: keyof typeof boundaries): void {
  boundaries[name].mockReset()
  boundaries[name].mockImplementation((...args: unknown[]) =>
    Reflect.apply(nativeFs[name] as (...args: unknown[]) => unknown, nativeFs, args)
  )
}

async function storeWithBudget(): Promise<GfsDownloadStore> {
  return withEnvironment({ MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT: '1' }, async () => {
    vi.resetModules()
    const { GfsDownloadStore: Store } = await import('./gfsDownloadStore')
    const store = new Store(hostRoot) as GfsDownloadStore
    stores.push(store)
    await store.initialize()
    return store
  })
}

function failTrashRemoval(): string[] {
  const hits: string[] = []
  boundaries.rm.mockImplementation((...args: unknown[]) => {
    if (path.basename(String(args[0])).startsWith('.trash-')) {
      hits.push(String(args[0]))
      return Promise.reject(eio())
    }
    return Reflect.apply(nativeFs.rm, nativeFs, args)
  })
  return hits
}

// Unpinned: two 4 MiB pins would exceed the Host-wide protected cap (7.5 MiB
// of the 10 MiB budget), and an expired copy is swept whether it is pinned or not.
async function expiredTrash(store: GfsDownloadStore) {
  await completedCopy(store, roots.a, 'a', 1, 4 * MIB)
  await completedCopy(store, roots.b, 'b', 2, 4 * MIB)
  vi.setSystemTime(Date.now() + 2 * HOUR)
  const hits = failTrashRemoval()
  expect((await store.cleanupExpired()).removeFailed).toBe(2)
  expect(hits).toHaveLength(2)
  expect(storedBytes(hostRoot)).toBe(8 * MIB)
  expect(view(store)).toMatchObject({ entries: 0, active: 0, held: 8 * MIB, charged: 8 * MIB })
  return hits
}

/** 8 MiB stay on disk, so a third 4 MiB admission would exceed the 10 MiB budget. */
async function expectRefused(store: GfsDownloadStore): Promise<void> {
  await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({
    code: 'host_quota_exceeded',
  })
}

/** Clears every fault, lets cleanup remove the leftovers, then publishes and reads a fresh copy. */
async function recoverAndWitness(store: GfsDownloadStore): Promise<void> {
  for (const name of ['rm', 'rename', 'realpath', 'open'] as const) restoreBoundary(name)
  await store.cleanupExpired()
  expect(storedBytes(hostRoot)).toBe(0)
  const fresh = await completedCopy(store, roots.c, 'c', 3, 4 * MIB, { owner: 'fresh' })
  expect((await store.readManagedFile(fresh.receipt.path, 'c')).equals(fresh.bytes)).toBe(true)
  expect(view(store).charged).toBe(4 * MIB)
}

beforeEach(async () => {
  nativeFs = await vi.importActual<typeof fs>('node:fs/promises')
  for (const name of Object.keys(boundaries) as Array<keyof typeof boundaries>)
    restoreBoundary(name)
  vi.useFakeTimers({ toFake: ['Date'] })
  vi.setSystemTime(new Date('2026-10-08T12:00:00Z'))
  hostRoot = await nativeFs.realpath(syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-removal-charge-')))
  roots = Object.fromEntries(['a', 'b', 'c', 'd'].map(id => [id, callerDirectory(hostRoot, id)]))
  boundaries.statfs.mockImplementation(async (target: string) => ({
    ...(await nativeFs.statfs(target, { bigint: true })),
    bsize: 1n,
    blocks: TOTAL,
    bavail: TOTAL - BigInt(storedBytes(hostRoot)),
  }))
})

afterEach(async () => {
  for (const release of releases.splice(0)) release()
  vi.restoreAllMocks()
  vi.useRealTimers()
  for (const store of stores.splice(0)) await store.close(0)
  await nativeFs.rm(hostRoot, { recursive: true, force: true })
})

describe('GFS download store removal charges', () => {
  it('RC-UNDO-FAIL: post-rename verification and undo failures retain the renamed bytes', async () => {
    const store = await storeWithBudget()
    // Unpinned, as in expiredTrash: the copies expire before the sweep.
    await completedCopy(store, roots.a, 'a', 1, 4 * MIB)
    await completedCopy(store, roots.b, 'b', 2, 4 * MIB)
    vi.setSystemTime(Date.now() + 2 * HOUR)
    const checkFault = new Set<string>()
    const undone: string[] = []
    boundaries.rename.mockImplementation(async (...args: unknown[]) => {
      const from = String(args[0])
      const to = String(args[1])
      if (path.basename(from).startsWith('.trash-') && path.basename(to).startsWith('input-')) {
        undone.push(from)
        throw eio()
      }
      const result = await Reflect.apply(nativeFs.rename, nativeFs, args)
      if (path.basename(from).startsWith('input-') && path.basename(to).startsWith('.trash-'))
        checkFault.add(path.dirname(to))
      return result
    })
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      if (checkFault.delete(String(args[0]))) throw eio()
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    failTrashRemoval()
    expect((await store.cleanupExpired()).removeFailed).toBe(2)
    expect(undone).toHaveLength(2)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    // The carried charges moved to the trash names in this sweep, not in a later one.
    expect(view(store)).toMatchObject({ entries: 0, held: 8 * MIB, charged: 8 * MIB })
    expect([...view(store).state.held.keys()].sort()).toEqual([...undone].sort())
    await expectRefused(store)
    await recoverAndWitness(store)
  })

  it.each(['partial', 'post-source-rename'] as const)(
    'RC-FAILED-TRANSFER-%s: failed transfer trash keeps its former reservation charged',
    async stage => {
      const store = await storeWithBudget()
      const hits = failTrashRemoval()
      // One transfer at a time: two 4 MiB reservations at once would exceed the
      // Host-wide protected cap (7.5 MiB of the 10 MiB budget). A held trash
      // charge is not protected, so the second transfer is admitted after the
      // first one failed.
      for (const [index, caller] of ['a', 'b'].entries()) {
        const attempt = await startTransfer(store, roots[caller], caller, index, 4 * MIB)
        if (stage === 'post-source-rename') {
          const directory = downloadDirectory(roots[caller], attempt.transfer.id)
          boundaries.open.mockImplementation(async (...args: unknown[]) => {
            const handle = (await Reflect.apply(nativeFs.open, nativeFs, args)) as fs.FileHandle
            if (String(args[0]) === directory) vi.spyOn(handle, 'sync').mockRejectedValueOnce(eio())
            return handle
          })
          await expect(
            store.publish(attempt.transfer.id, caller, digestOf(attempt.bytes))
          ).rejects.toMatchObject({ code: 'storage_write_failed' })
          expect(syncFs.existsSync(path.join(directory, 'source'))).toBe(true)
          expect(syncFs.existsSync(path.join(directory, 'meta.json'))).toBe(true)
          restoreBoundary('open')
        }
        expect(view(store).active).toBe(4 * MIB)
        await expect(store.fail(attempt.transfer.id, caller)).rejects.toMatchObject({
          code: 'storage_write_failed',
        })
      }
      // Three removals: a's trash, a's trash again under a new trash name
      // (retried by b's admission sweep), then b's trash.
      expect(hits).toHaveLength(3)
      expect(storedBytes(hostRoot)).toBe(8 * MIB)
      // Each reservation moved, unchanged, to the last trash name its removal left.
      expect(view(store)).toMatchObject({ active: 0, held: 8 * MIB, charged: 8 * MIB })
      expect([...view(store).state.held.keys()].sort()).toEqual(hits.slice(1).sort())
      await expectRefused(store)
      await recoverAndWitness(store)
    }
  )

  it('RC-NO-DOUBLE-CHARGE: repeated trash failures charge exactly once and admission at the remainder works', async () => {
    const store = await storeWithBudget()
    await expiredTrash(store)
    for (let i = 0; i < 4; i++) {
      expect((await store.cleanupExpired()).removeFailed).toBe(2)
      expect(view(store)).toMatchObject({ held: 8 * MIB, charged: 8 * MIB })
      expect(storedBytes(hostRoot)).toBe(8 * MIB)
    }
    await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })
    const fits = await startTransfer(store, roots.c, 'c', 4, 2 * MIB)
    expect(view(store).charged).toBe(10 * MIB)
    expect(storedBytes(hostRoot)).toBe(10 * MIB)
    restoreBoundary('rm')
    await store.fail(fits.transfer.id, 'c')
    await store.cleanupExpired()
    expect(view(store).charged).toBe(0)
    await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).resolves.toBeDefined()
  })

  it.each(['file', 'missing', 'parent-file'] as const)(
    'RC-TRASH-ABSENCE-%s: definitive absence releases a held trash charge despite a failed listing',
    async replacement => {
      const store = await storeWithBudget()
      const hits = await expiredTrash(store)
      await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({
        code: 'host_quota_exceeded',
      })
      const trash = [...view(store).state.held.keys()].find(p =>
        p.includes(`${path.sep}a${path.sep}`)
      )!
      expect(hits.length).toBeGreaterThanOrEqual(4)
      if (replacement === 'parent-file') {
        syncFs.rmSync(path.dirname(trash), { recursive: true, force: true })
        syncFs.writeFileSync(path.dirname(trash), '', { mode: 0o600 })
        await expect(nativeFs.lstat(trash)).rejects.toMatchObject({ code: 'ENOTDIR' })
      } else {
        syncFs.rmSync(trash, { recursive: true, force: true })
        if (replacement === 'file') syncFs.writeFileSync(trash, '', { mode: 0o600 })
        else await expect(nativeFs.lstat(trash)).rejects.toMatchObject({ code: 'ENOENT' })
      }
      const listed: string[] = []
      boundaries.readdir.mockImplementation((...args: unknown[]) => {
        if (String(args[0]) === path.join(roots.a, '.gfs-downloads')) {
          listed.push(String(args[0]))
          return Promise.reject(eio())
        }
        return Reflect.apply(nativeFs.readdir, nativeFs, args)
      })
      const fits = await startTransfer(store, roots.c, 'c', 3, 4 * MIB)
      expect(view(store)).toMatchObject({ held: 4 * MIB, active: 4 * MIB, charged: 8 * MIB })
      expect(storedBytes(hostRoot)).toBe(8 * MIB)
      if (replacement !== 'parent-file') expect(listed.length).toBeGreaterThan(0)
      restoreBoundary('rm')
      await store.fail(fits.transfer.id, 'c')
    }
  )

  it('RC-TRASH-UNKNOWN: failed listing and EIO probing retain trash charges until cleanup recovers', async () => {
    const store = await storeWithBudget()
    await expiredTrash(store)
    const trash = new Set(view(store).state.held.keys())
    const probed: string[] = []
    boundaries.readdir.mockImplementation((...args: unknown[]) => {
      if (String(args[0]).endsWith('.gfs-downloads')) return Promise.reject(eio())
      return Reflect.apply(nativeFs.readdir, nativeFs, args)
    })
    boundaries.lstat.mockImplementation((...args: unknown[]) => {
      if (trash.has(String(args[0]))) {
        probed.push(String(args[0]))
        return Promise.reject(eio())
      }
      return Reflect.apply(nativeFs.lstat, nativeFs, args)
    })
    await expect(startTransfer(store, roots.c, 'c', 3, 4 * MIB)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })
    expect(probed).toHaveLength(2)
    expect(view(store).held).toBe(8 * MIB)
    for (const name of ['readdir', 'lstat', 'rm'] as const) restoreBoundary(name)
    expect((await store.cleanupExpired()).removedIncomplete).toBe(2)
    expect(view(store).charged).toBe(0)
    const fresh = await completedCopy(store, roots.c, 'c', 3, 4 * MIB)
    expect((await store.readManagedFile(fresh.receipt.path, 'c')).equals(fresh.bytes)).toBe(true)
  })

  it('RC-SERIAL-SWEEPS: concurrent admission and cleanup queue behind the pre-check sweep and preserve live pins and active transfers', async () => {
    const store = await storeWithBudget()
    const expiry = new Date(Date.now() + 24 * HOUR).toISOString()
    // Protected bytes stay within the Host-wide cap (7.5 MiB of the 10 MiB
    // budget): the live pin and the active transfer take 1 MiB each, and the
    // expiring copy and the admission below 3 MiB each, one after the other.
    const live = await completedCopy(store, roots.a, 'a', 0, MIB, {
      owner: 'live',
      expiresAt: expiry,
    })
    const active = await startTransfer(store, roots.a, 'a', 1, MIB, { expiresAt: expiry })
    const expiring = await completedCopy(store, roots.b, 'b', 2, 3 * MIB)
    vi.setSystemTime(Date.now() + 2 * HOUR)
    let enter!: () => void
    let release!: () => void
    const entered = new Promise<void>(resolve => {
      enter = resolve
    })
    const released = new Promise<void>(resolve => {
      release = resolve
    })
    releases.push(release)
    let armed = true
    boundaries.rm.mockImplementation(async (...args: unknown[]) => {
      if (armed && path.basename(String(args[0])).startsWith('.trash-')) {
        armed = false
        enter()
        await released
      }
      return Reflect.apply(nativeFs.rm, nativeFs, args)
    })
    boundaries.statfs.mockClear()
    const first = startTransfer(store, roots.c, 'c', 3, 3 * MIB)
    await entered
    let secondSettled = false
    let cleanupSettled = false
    const second = startTransfer(store, roots.d, 'd', 4, 4 * MIB).then(
      value => {
        secondSettled = true
        return value
      },
      error => {
        secondSettled = true
        return error as { code: string }
      }
    )
    const cleanup = store.cleanupExpired().then(value => {
      cleanupSettled = true
      return value
    })
    await new Promise<void>(resolve => setImmediate(resolve))
    expect(secondSettled).toBe(false)
    expect(cleanupSettled).toBe(false)
    expect(boundaries.statfs).not.toHaveBeenCalled()
    release()
    const admitted = await first
    await expect(second).resolves.toMatchObject({ code: 'download_busy' })
    await cleanup
    expect(secondSettled).toBe(true)
    expect(cleanupSettled).toBe(true)
    expect(syncFs.existsSync(downloadDirectory(roots.b, expiring.receipt.id))).toBe(false)
    expect(
      syncFs.readFileSync(path.join(roots.a, active.transfer.partialPath)).equals(active.bytes)
    ).toBe(true)
    expect((await store.readManagedFile(live.receipt.path, 'a')).equals(live.bytes)).toBe(true)
    await expect(store.readManagedFile(live.receipt.path, 'd')).rejects.toMatchObject({
      code: 'download_missing',
      message: 'GFS download store failed (download_missing)',
    })
    expect(view(store).charged).toBe(5 * MIB)
    await store.fail(active.transfer.id, 'a')
    const next = await startTransfer(store, roots.d, 'd', 4, MIB)
    expect(view(store).charged).toBe(5 * MIB)
    expect((await store.readManagedFile(live.receipt.path, 'a')).equals(live.bytes)).toBe(true)
    await store.fail(admitted.transfer.id, 'c')
    await store.fail(next.transfer.id, 'd')
  })

  it.each([false, true])(
    'RC-REUSE-UNPINNED-%s: crossing the one-hour reuse margin after the hash is a miss with and without a new retention owner',
    async owner => {
      const store = await storeWithBudget()
      const copy = await completedCopy(store, roots.a, 'a', 0, 1024)
      await expect(store.reusableReceipt('a', sourceFor(0), 1024)).resolves.toMatchObject({
        id: copy.receipt.id,
      })
      const reuseMargin = 60 * 60_000
      vi.setSystemTime(Date.parse(copy.receipt.expiresAt) - reuseMargin - 1)
      let enter!: () => void
      let release!: () => void
      const entered = new Promise<void>(resolve => {
        enter = resolve
      })
      const released = new Promise<void>(resolve => {
        release = resolve
      })
      releases.push(release)
      let armed = true
      boundaries.open.mockImplementation(async (...args: unknown[]) => {
        const handle = (await Reflect.apply(nativeFs.open, nativeFs, args)) as fs.FileHandle
        if (armed && String(args[0]).endsWith(`${path.sep}source`)) {
          armed = false
          const read = handle.read.bind(handle)
          vi.spyOn(handle, 'read').mockImplementationOnce(async (...readArgs: unknown[]) => {
            enter()
            await released
            return Reflect.apply(read, handle, readArgs)
          })
        }
        return handle
      })
      const reuse = store.reusableReceipt(
        'a',
        sourceFor(0),
        1024,
        owner ? { retentionOwnerId: 'new-owner' } : undefined
      )
      await entered
      vi.setSystemTime(Date.parse(copy.receipt.expiresAt) - reuseMargin)
      release()
      await expect(reuse).resolves.toBeUndefined()
      expect(view(store).state.pins.size).toBe(0)
      vi.setSystemTime(Date.parse(copy.receipt.expiresAt))
      expect((await store.cleanupExpired()).removedExpired).toBe(1)
      const fresh = await completedCopy(store, roots.a, 'a', 0, 1024, { owner: 'new-owner' })
      expect((await store.readManagedFile(fresh.receipt.path, 'a')).equals(fresh.bytes)).toBe(true)
    }
  )

  it('RC-RESTART-TRASH: trash that survives startup cleanup remains charged after reopening the store', async () => {
    const original = await storeWithBudget()
    const hits = await expiredTrash(original)
    await original.close(0)
    const reopened = await storeWithBudget()
    expect(hits).toHaveLength(4)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    // Measured once on restart: both trees' regular files, meta.json included.
    expect(regularBytes(hostRoot)).toBeGreaterThan(8 * MIB)
    expect(view(reopened).held).toBe(regularBytes(hostRoot))
    await expectRefused(reopened)
    await recoverAndWitness(reopened)
  })

  it('RC-FAIL-BEFORE-RENAME: a failed transfer still in its original incomplete directory retains its charge', async () => {
    const store = await storeWithBudget()
    const hits: string[] = []
    boundaries.rename.mockImplementation((...args: unknown[]) => {
      if (path.basename(String(args[0])).startsWith('input-')) {
        hits.push(String(args[0]))
        return Promise.reject(eio())
      }
      return Reflect.apply(nativeFs.rename, nativeFs, args)
    })
    // One transfer at a time: two 4 MiB reservations at once would exceed the
    // Host-wide protected cap (7.5 MiB of the 10 MiB budget).
    const a = await startTransfer(store, roots.a, 'a', 1, 4 * MIB)
    await expect(store.fail(a.transfer.id, 'a')).rejects.toMatchObject({
      code: 'storage_write_failed',
    })
    expect(hits).toHaveLength(1)
    // b's admission sweep retries a's directory once more.
    const b = await startTransfer(store, roots.b, 'b', 2, 4 * MIB)
    expect(hits).toHaveLength(2)
    await expect(store.fail(b.transfer.id, 'b')).rejects.toMatchObject({
      code: 'storage_write_failed',
    })
    expect(hits).toHaveLength(3)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    expect(view(store).held).toBe(8 * MIB)
    await expectRefused(store)
    expect(hits).toHaveLength(5)
    await recoverAndWitness(store)
  })

  it.each([false, true])(
    'RC-VOLUME-SECOND-CHECK-%s: the post-reclaim measurement retries invalid readings and retains the correct error',
    async persistent => {
      const store = await storeWithBudget()
      const valid = {
        ...(await nativeFs.statfs(hostRoot, { bigint: true })),
        bsize: 1n,
        blocks: TOTAL,
        bavail: TOTAL,
      }
      const invalid = { ...valid, bavail: -1n }
      boundaries.statfs.mockReset()
      boundaries.statfs
        .mockResolvedValueOnce(valid)
        .mockResolvedValueOnce(invalid)
        .mockResolvedValueOnce(persistent ? invalid : valid)
      const before = await quotaCount('host', 'free_space')
      const budgetBefore = await quotaCount('host', 'storage_bytes')
      const outcome = await startTransfer(store, roots.a, 'a', 1, 1024).then(
        admitted => ({ admitted }),
        error => error as { code: string; message: string }
      )
      expect(boundaries.statfs).toHaveBeenCalledTimes(3)
      expect(await quotaCount('host', 'storage_bytes')).toBe(budgetBefore)
      if (persistent) {
        expect(outcome).toMatchObject({
          code: 'volume_unmeasurable',
          message: 'GFS download store failed (volume_unmeasurable)',
        })
        expect(await quotaCount('host', 'free_space')).toBe(before + 1)
        expect(view(store).active).toBe(0)
      } else {
        expect('admitted' in outcome).toBe(true)
        expect(await quotaCount('host', 'free_space')).toBe(before)
        if ('admitted' in outcome) await store.fail(outcome.admitted.transfer.id, 'a')
      }
      boundaries.statfs.mockResolvedValue(valid)
      const fresh = await startTransfer(store, roots.a, 'a', 1, 1024)
      await store.fail(fresh.transfer.id, 'a')
    }
  )

  it.each([false, true])(
    'RC-VOLUME-REUSE-%s: a new retention owner shares the invalid-statfs retry and category',
    async persistent => {
      const store = await storeWithBudget()
      const copy = await completedCopy(store, roots.a, 'a', 0, 1024)
      const valid = {
        ...(await nativeFs.statfs(hostRoot, { bigint: true })),
        bsize: 1n,
        blocks: TOTAL,
        bavail: TOTAL,
      }
      boundaries.statfs.mockReset()
      boundaries.statfs
        .mockResolvedValueOnce({ ...valid, bsize: 0n })
        .mockResolvedValueOnce(persistent ? { ...valid, bsize: 0n } : valid)
      const before = await quotaCount('host', 'free_space')
      const outcome = await store
        .reusableReceipt('a', sourceFor(0), 1024, { retentionOwnerId: 'new' })
        .then(
          receipt => ({ receipt }),
          error => error as { code: string; message: string }
        )
      expect(boundaries.statfs).toHaveBeenCalledTimes(2)
      if (persistent) {
        expect(outcome).toMatchObject({
          code: 'volume_unmeasurable',
          message: 'GFS download store failed (volume_unmeasurable)',
        })
        expect(await quotaCount('host', 'free_space')).toBe(before + 1)
        expect(view(store).state.pins.size).toBe(0)
      } else {
        expect(outcome).toMatchObject({ receipt: { id: copy.receipt.id } })
        expect(await quotaCount('host', 'free_space')).toBe(before)
        expect(view(store).state.pins.size).toBe(1)
      }
      boundaries.statfs.mockResolvedValue(valid)
      await expect(
        store.reusableReceipt('a', sourceFor(0), 1024, { retentionOwnerId: 'new' })
      ).resolves.toMatchObject({ id: copy.receipt.id })
      expect((await store.readManagedFile(copy.receipt.path, 'a')).equals(copy.bytes)).toBe(true)
    }
  )

  it.each([false, true])(
    'RC-VOLUME-EIO-%s: a rejected statfs propagates without treating it as an invalid reading',
    async invalidFirst => {
      const store = await storeWithBudget()
      const valid = {
        ...(await nativeFs.statfs(hostRoot, { bigint: true })),
        bsize: 1n,
        blocks: TOTAL,
        bavail: TOTAL,
      }
      boundaries.statfs.mockReset()
      if (invalidFirst) boundaries.statfs.mockResolvedValueOnce({ ...valid, blocks: 0n })
      boundaries.statfs.mockRejectedValueOnce(eio())
      const before = await quotaCount('host', 'free_space')
      await expect(startTransfer(store, roots.a, 'a', 1, 1024)).rejects.toMatchObject({
        code: 'EIO',
        message: 'EIO: injected fault',
      })
      expect(boundaries.statfs).toHaveBeenCalledTimes(invalidFirst ? 2 : 1)
      expect(await quotaCount('host', 'free_space')).toBe(before)
      boundaries.statfs.mockResolvedValue(valid)
      const admitted = await startTransfer(store, roots.a, 'a', 1, 1024)
      await store.fail(admitted.transfer.id, 'a')
    }
  )

  it('RC-FAIL-PARENT-REFUSED: a failed transfer whose parent check refuses before the rename keeps its reservation charged', async () => {
    const store = await storeWithBudget()
    const faulted = new Set<string>()
    const hits: string[] = []
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      if (faulted.has(String(args[0]))) {
        hits.push(String(args[0]))
        throw eio()
      }
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    // One transfer at a time: two 4 MiB reservations at once would exceed the
    // Host-wide protected cap (7.5 MiB of the 10 MiB budget). Each parent is
    // faulted after its own admission, which resolves it.
    for (const [index, caller] of ['a', 'b'].entries()) {
      const attempt = await startTransfer(store, roots[caller], caller, index + 1, 4 * MIB)
      faulted.add(path.join(roots[caller], '.gfs-downloads'))
      await expect(store.fail(attempt.transfer.id, caller)).rejects.toMatchObject({
        code: 'storage_write_failed',
      })
    }
    expect(hits.length).toBeGreaterThanOrEqual(2)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    expect(view(store)).toMatchObject({ entries: 0, active: 0, held: 8 * MIB, charged: 8 * MIB })
    await expectRefused(store)
    await recoverAndWitness(store)
  })

  it('RC-SYMLINK-ROOT: a .gfs-downloads symlink that cannot be removed is never walked into', async () => {
    const store = await storeWithBudget()
    const copy = await completedCopy(store, roots.b, 'b', 2, 4 * MIB, {
      owner: 'b-owner',
      expiresAt: new Date(Date.now() + 24 * HOUR).toISOString(),
    })
    const link = path.join(roots.a, '.gfs-downloads')
    syncFs.rmSync(link, { recursive: true, force: true })
    syncFs.symlinkSync(path.join(roots.b, '.gfs-downloads'), link)
    const renames: string[] = []
    boundaries.rename.mockImplementation((...args: unknown[]) => {
      if (String(args[0]) === link) {
        renames.push(String(args[0]))
        return Promise.reject(Object.assign(new Error('EACCES'), { code: 'EACCES' }))
      }
      return Reflect.apply(nativeFs.rename, nativeFs, args)
    })
    const walked: string[] = []
    boundaries.readdir.mockImplementation((...args: unknown[]) => {
      const target = String(args[0])
      if (target === link || target.startsWith(link + path.sep)) walked.push(target)
      return Reflect.apply(nativeFs.readdir, nativeFs, args)
    })
    for (let i = 0; i < 3; i++) await store.cleanupExpired()
    // Two per sweep: an EACCES rename restores the owner's access on the
    // caller root and is retried once, and the retry fails the same way.
    expect(renames).toHaveLength(6)
    expect(walked).toEqual([])
    expect(view(store)).toMatchObject({ held: 0, charged: 4 * MIB })
    expect((await store.readManagedFile(copy.receipt.path, 'b')).equals(copy.bytes)).toBe(true)
  })

  it('RC-REFUSED-TRASH-NOT-READ: a trash left in a refused parent with no charge in hand is not measured', async () => {
    const original = await storeWithBudget()
    await expiredTrash(original)
    await original.close(0)
    const downloads = new Set([
      path.join(roots.a, '.gfs-downloads'),
      path.join(roots.b, '.gfs-downloads'),
    ])
    const renamedInto = new Set<string>()
    const undone: string[] = []
    boundaries.rename.mockImplementation(async (...args: unknown[]) => {
      const from = String(args[0])
      const to = String(args[1])
      if (renamedInto.has(from)) {
        undone.push(from)
        throw eio()
      }
      const result = await Reflect.apply(nativeFs.rename, nativeFs, args)
      if (downloads.has(path.dirname(to))) renamedInto.add(to)
      return result
    })
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      const target = String(args[0])
      if (downloads.has(target) && renamedInto.size > undone.length) throw eio()
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    const walked: string[] = []
    boundaries.readdir.mockImplementation((...args: unknown[]) => {
      if (renamedInto.has(String(args[0]))) walked.push(String(args[0]))
      return Reflect.apply(nativeFs.readdir, nativeFs, args)
    })
    const reopened = await storeWithBudget()
    expect(undone).toHaveLength(2)
    expect(walked).toEqual([])
    for (const trash of undone) expect(view(reopened).state.held.has(trash)).toBe(false)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    await recoverAndWitness(reopened)
  })

  it('RC-ROOT-REFUSAL-CODE: a replaced .gfs-downloads whose undo fails reports the refusal and keeps its copies charged on the trash', async () => {
    const store = await storeWithBudget()
    await completedCopy(store, roots.a, 'a', 1, 4 * MIB, { owner: 'keep' })
    const downloads = path.join(roots.a, '.gfs-downloads')
    syncFs.chmodSync(downloads, 0o755)
    let trash: string | undefined
    boundaries.rename.mockImplementation(async (...args: unknown[]) => {
      const from = String(args[0])
      if (from === downloads) {
        trash = String(args[1])
        return Reflect.apply(nativeFs.rename, nativeFs, args)
      }
      if (trash !== undefined && from === trash) throw eio()
      return Reflect.apply(nativeFs.rename, nativeFs, args)
    })
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      if (trash !== undefined && String(args[0]) === roots.a) return '/elsewhere'
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    await expect(startTransfer(store, roots.a, 'a', 2, 1024)).rejects.toMatchObject({
      code: 'workspace_unavailable',
    })
    expect(trash).toBeDefined()
    expect(view(store).state.held.get(trash!)?.sizeBytes).toBe(4 * MIB)
    expect(view(store)).toMatchObject({ entries: 0, charged: 4 * MIB })
  })

  it('RC-INDEXED-KEPT: an evicted entry whose rename fails stays indexed and is charged once', async () => {
    const store = await storeWithBudget()
    await completedCopy(store, roots.a, 'a', 1, 4 * MIB)
    await completedCopy(store, roots.b, 'b', 2, 4 * MIB)
    const hits: string[] = []
    boundaries.rename.mockImplementation((...args: unknown[]) => {
      const source = String(args[0])
      const cached = source.startsWith(roots.a) || source.startsWith(roots.b)
      if (cached && path.basename(source).startsWith('input-')) {
        hits.push(source)
        return Promise.reject(eio())
      }
      return Reflect.apply(nativeFs.rename, nativeFs, args)
    })
    // The plan evicts one of the two copies (the frozen clock ties them); its
    // rename fails, so the admission still exceeds the budget.
    for (let i = 0; i < 3; i++) {
      await expect(startTransfer(store, roots.c, 'c', 3 + i, 4 * MIB)).rejects.toMatchObject({
        code: 'host_quota_exceeded',
      })
      expect(view(store)).toMatchObject({ entries: 8 * MIB, held: 0, active: 0, charged: 8 * MIB })
    }
    expect(hits).toHaveLength(3)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    restoreBoundary('rename')
    const fits = await startTransfer(store, roots.c, 'c', 6, 4 * MIB)
    // One copy is evicted now; the other and c's partial are what remains.
    expect(view(store)).toMatchObject({ entries: 4 * MIB, held: 0, active: 4 * MIB })
    expect(storedBytes(roots.a) + storedBytes(roots.b)).toBe(4 * MIB)
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    await store.fail(fits.transfer.id, 'c')
  })

  it('RC-EXPIRED-KEPT: an expired entry whose removal fails in the sweep stays indexed and charged', async () => {
    const store = await storeWithBudget()
    const expiresAt = new Date(Date.now() + HOUR).toISOString()
    const copyA = await completedCopy(store, roots.a, 'a', 1, 4 * MIB, { expiresAt })
    const copyB = await completedCopy(store, roots.b, 'b', 2, 4 * MIB, { expiresAt })
    const stuck = [copyA, copyB].map(({ receipt }, index) =>
      downloadDirectory(roots[index === 0 ? 'a' : 'b'], receipt.id)
    )
    vi.setSystemTime(Date.now() + 2 * HOUR)
    const attempts: string[] = []
    boundaries.rename.mockImplementation((...args: unknown[]) => {
      if (stuck.includes(String(args[0]))) {
        attempts.push(String(args[0]))
        return Promise.reject(eio())
      }
      return Reflect.apply(nativeFs.rename, nativeFs, args)
    })
    const removeFailed = await expiryCount('remove_failed')
    const removedExpired = await expiryCount('expired_removed')

    expect(await store.cleanupExpired()).toMatchObject({ removedExpired: 0, removeFailed: 2 })

    // Witness: the sweep tried to remove both expired copies, and counted it.
    expect([...attempts].sort()).toEqual([...stuck].sort())
    expect(await expiryCount('remove_failed')).toBe(removeFailed + 2)
    expect(await expiryCount('expired_removed')).toBe(removedExpired)
    // Both are still indexed under their own names, and their 8 MiB charged.
    expect([...view(store).state.entries.keys()].sort()).toEqual(
      [copyA.receipt.id, copyB.receipt.id].sort()
    )
    expect(view(store)).toMatchObject({ entries: 8 * MIB, held: 0, charged: 8 * MIB })
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    // The charge still counts against the 10 MiB budget: 8 + 4 MiB is refused.
    const denied = await quotaCount('host', 'storage_bytes')
    await expectRefused(store)
    expect(await quotaCount('host', 'storage_bytes')).toBe(denied + 1)
    restoreBoundary('rename')
    await recoverAndWitness(store)
  })

  it('RC-UNDO-RESTORED-UNINDEXED: startup leftovers restored by an undone refusal are measured once their parent passes again', async () => {
    for (const caller of ['a', 'b'])
      plantDownload(roots[caller], { bytes: Buffer.alloc(4 * MIB), meta: 'omit' })
    // The check after each rename to a trash name fails once with EIO; the
    // rename back succeeds, so each leftover is under its own name again.
    const refused = new Set<string>()
    const undone: string[] = []
    boundaries.rename.mockImplementation(async (...args: unknown[]) => {
      const [from, to] = [String(args[0]), String(args[1])]
      const result = await Reflect.apply(nativeFs.rename, nativeFs, args)
      if (path.basename(from).startsWith('.trash-')) undone.push(to)
      else if (path.basename(to).startsWith('.trash-')) refused.add(path.dirname(to))
      return result
    })
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      if (refused.delete(String(args[0]))) throw eio()
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    const store = await storeWithBudget()
    expect(undone).toHaveLength(2)
    expect(view(store)).toMatchObject({ entries: 0, held: 8 * MIB, charged: 8 * MIB })
    expect([...view(store).state.held.keys()].sort()).toEqual([...undone].sort())
    expect(storedBytes(hostRoot)).toBe(8 * MIB)
    // Each admission retries both removals, which are undone again.
    await expectRefused(store)
    expect(undone).toHaveLength(4)
    await recoverAndWitness(store)
  })

  it('RC-WALK-SWAP: a directory replaced by a symlink while a leftover is measured is not walked', async () => {
    const planted = plantDownload(roots.a, { bytes: Buffer.alloc(4 * MIB), meta: 'omit' })
    syncFs.mkdirSync(path.join(planted.directory, 'nested'))
    const victim = path.join(roots.b, 'ordinary-workspace-data')
    syncFs.mkdirSync(victim)
    const victimFile = path.join(victim, 'outside.bin')
    syncFs.writeFileSync(victimFile, Buffer.alloc(8 * MIB, 29))
    const swapped: string[] = []
    const walked: string[] = []
    boundaries.readdir.mockImplementation(async (...args: unknown[]) => {
      const target = String(args[0])
      walked.push(target)
      const listing = await Reflect.apply(nativeFs.readdir, nativeFs, args)
      // After the trash is listed, its child directory becomes a symlink.
      if (path.basename(target).startsWith('.trash-') && swapped.length === 0) {
        const nested = path.join(target, 'nested')
        syncFs.rmdirSync(nested)
        syncFs.symlinkSync(victim, nested)
        swapped.push(nested)
      }
      return listing
    })
    failTrashRemoval()
    const store = await storeWithBudget()
    expect(swapped).toHaveLength(1)
    expect(walked).not.toContain(swapped[0])
    expect(walked).not.toContain(victim)
    expect(view(store)).toMatchObject({ entries: 0, held: 4 * MIB, charged: 4 * MIB })
    // 4 MiB held plus 4 MiB requested fit the 10 MiB budget.
    const fits = await startTransfer(store, roots.c, 'c', 3, 4 * MIB)
    expect(view(store)).toMatchObject({ held: 4 * MIB, active: 4 * MIB })
    restoreBoundary('rm')
    await store.fail(fits.transfer.id, 'c')
    restoreBoundary('readdir')
    await recoverAndWitness(store)
    expect(syncFs.statSync(victimFile).size).toBe(8 * MIB)
  })

  it('RC-REFUSED-IN-PLACE-NOT-READ: a leftover whose parent check keeps failing is not walked to measure it', async () => {
    plantDownload(roots.a, { bytes: Buffer.alloc(4 * MIB), meta: 'omit' })
    const downloads = path.join(roots.a, '.gfs-downloads')
    const checks: string[] = []
    boundaries.realpath.mockImplementation(async (...args: unknown[]) => {
      if (String(args[0]) === downloads) {
        checks.push(downloads)
        throw eio()
      }
      return Reflect.apply(nativeFs.realpath, nativeFs, args)
    })
    const walked: string[] = []
    boundaries.readdir.mockImplementation((...args: unknown[]) => {
      if (path.basename(String(args[0])).startsWith('input-')) walked.push(String(args[0]))
      return Reflect.apply(nativeFs.readdir, nativeFs, args)
    })
    const store = await storeWithBudget()
    expect((await store.cleanupExpired()).removeFailed).toBe(1)
    // Refused before the rename and again by the recheck, in both sweeps.
    expect(checks.length).toBeGreaterThanOrEqual(4)
    expect(walked).toEqual([])
    expect(view(store)).toMatchObject({ entries: 0, held: 0 })
    expect(storedBytes(hostRoot)).toBe(4 * MIB)
    await recoverAndWitness(store)
  })
})