Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
11.8 KB
Raw Blame
/**
 * R2-F2: a duplicate whose removal failed stays charged against the Host
 * budget until its removal succeeds or `lstat` proves it gone, including
 * through sweeps that cannot inspect or list it. Ported from the final
 * review's `SOL-R2-HELD-meta-open` and `SOL-R2-HELD-caller-list` repros to the
 * byte budget. Every case runs on a real temporary directory; faults are
 * injected only on the operation that fails.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import {
  callerDirectory,
  downloadDirectory,
  exists,
  metaFor,
  plantDownload,
  startTransfer,
} from '../__tests__/fixtures/gfsStoreTestKit'
import { GfsDownloadStore } from './gfsDownloadStore'

type FsOperation = 'open' | 'lstat' | 'rename' | 'readdir'

const { faults } = vi.hoisted(() => ({
  faults: new Map<string, (target: string) => string | undefined>(),
}))

// A 115-byte volume at the default 70% gives a budget of floor(80.5) = 80:
// eight 5-byte copies per caller fill it exactly.
const VOLUME_BYTES = 115n

vi.mock('node:fs/promises', async original => {
  const actual = await original<typeof fs>()
  const wrap =
    <F extends (...args: never[]) => unknown>(operation: string, real: F) =>
    (...args: Parameters<F>) => {
      const code = faults.get(operation)?.(String(args[0]))
      if (code !== undefined) {
        const error = new Error(`${code}: injected`) as NodeJS.ErrnoException
        error.code = code
        return Promise.reject(error)
      }
      return real(...args)
    }
  return {
    ...actual,
    open: wrap('open', actual.open),
    lstat: wrap('lstat', actual.lstat),
    rename: wrap('rename', actual.rename),
    readdir: wrap('readdir', actual.readdir),
    statfs: async (target: string, options: { bigint: true }) => {
      const real = await actual.statfs(target, options)
      return { ...real, bsize: 1n, blocks: VOLUME_BYTES, bavail: real.bavail * real.bsize }
    },
  }
})

/** Records every `operation` call that `matches`, failing it with EIO. */
function failWithEio(operation: FsOperation, matches: (target: string) => boolean): string[] {
  const calls: string[] = []
  faults.set(operation, target => {
    if (!matches(target)) return undefined
    calls.push(target)
    return 'EIO'
  })
  return calls
}

const OWNER = 'caller-b'
const PLANTER = 'caller-a'
const DOWNLOADS = `${path.sep}users${path.sep}${PLANTER}${path.sep}.gfs-downloads`

let hostRoot: string
const stores: GfsDownloadStore[] = []

async function openStore(): Promise<GfsDownloadStore> {
  const opened = new GfsDownloadStore(hostRoot)
  stores.push(opened)
  await opened.initialize()
  return opened
}

function plantCopies(root: string, ids: string[]): void {
  const bytes = Buffer.from('quota')
  for (const id of ids) plantDownload(root, { id, bytes, meta: metaFor(id, bytes, Date.now()) })
}

/** Renames of the given ids' input directories fail; `only` limits it to one caller root. */
function failRemovals(ids: string[], only?: string): string[] {
  const names = new Set(ids.map(id => `input-${id}`))
  return failWithEio(
    'rename',
    target => names.has(path.basename(target)) && (only === undefined || target.includes(only))
  )
}

/**
 * The owner's 8 copies are indexed at startup, the planter's 8 duplicates of
 * the same ids appear afterwards, and every removal of either fails: 80 bytes
 * on disk against a budget of 80, so a 5-byte admission must be refused.
 */
async function heldDuplicates() {
  const owner = callerDirectory(hostRoot, OWNER)
  const planter = callerDirectory(hostRoot, PLANTER)
  const ids = Array.from({ length: 8 }, () => randomUUID())
  plantCopies(owner, ids)
  const store = await openStore()
  const renames = failRemovals(ids)
  plantCopies(planter, ids)
  const first = await store.cleanupExpired()
  // Witness: both copies of every id were removal attempts, and each failed.
  expect(first.removeFailed).toBe(ids.length * 2)
  expect(renames.length).toBeGreaterThanOrEqual(ids.length * 2)
  await expect(startTransfer(store, planter, PLANTER, 100, 5)).rejects.toMatchObject({
    code: 'host_quota_exceeded',
  })
  return { store, owner, planter, ids }
}

beforeEach(() => {
  faults.clear()
  hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-held-'))
})

afterEach(async () => {
  faults.clear()
  vi.restoreAllMocks()
  for (const opened of stores.splice(0)) await opened.close(0)
  await fs.rm(hostRoot, { recursive: true, force: true })
})

describe('GFS download store: held duplicate charges survive transient failures (R2-F2)', () => {
  it('SOL-R2-HELD-meta-open: a duplicate whose meta.json cannot be opened keeps its charge', async () => {
    const { store, planter, ids } = await heldDuplicates()
    const reads = failWithEio(
      'open',
      target => target.includes(`${DOWNLOADS}${path.sep}`) && path.basename(target) === 'meta.json'
    )

    // The sweep inside this admission inspects every duplicate and cannot.
    await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })
    // Witness: the inspection was attempted and failed.
    expect(reads.length).toBeGreaterThan(0)
    for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)

    // Inspection back, removals still failing: still refused.
    faults.delete('open')
    await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })

    // Removals back: the sweep removes every duplicate and capacity returns.
    faults.delete('rename')
    await store.cleanupExpired()
    for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)
    const admitted = await startTransfer(store, planter, PLANTER, 103, 5)
    await store.fail(admitted.transfer.id, PLANTER)
  })

  it('SOL-R2-HELD-caller-list: a duplicate whose .gfs-downloads cannot be listed keeps its charge', async () => {
    const { store, planter, ids } = await heldDuplicates()
    const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))

    await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })
    // Witness: the listing was attempted and failed.
    expect(listings.length).toBeGreaterThan(0)
    for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(true)

    faults.delete('readdir')
    await expect(startTransfer(store, planter, PLANTER, 102, 5)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })

    faults.delete('rename')
    await store.cleanupExpired()
    for (const id of ids) expect(exists(downloadDirectory(planter, id))).toBe(false)
    const admitted = await startTransfer(store, planter, PLANTER, 103, 5)
    await store.fail(admitted.transfer.id, PLANTER)
  })

  it('a carried charge is dropped once lstat reports the directory gone', async () => {
    const { store, planter, ids } = await heldDuplicates()
    const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
    // The duplicates disappear while their parent cannot be listed.
    for (const id of ids)
      syncFs.rmSync(downloadDirectory(planter, id), { recursive: true, force: true })

    // Only the owner's 40 bytes remain on disk: 40 + 5 fits the budget of 80.
    const admitted = await startTransfer(store, planter, PLANTER, 101, 5)
    // Witness: the sweep inside the admission could not list the duplicates,
    // so only the lstat re-check can have released their charge.
    expect(listings.length).toBeGreaterThan(0)
    await store.fail(admitted.transfer.id, PLANTER)
  })

  it('a carried charge whose lstat fails with anything but ENOENT or ENOTDIR is kept', async () => {
    const { store, planter, ids } = await heldDuplicates()
    const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
    const probes = failWithEio('lstat', target =>
      ids.some(id => target.endsWith(`${DOWNLOADS}${path.sep}input-${id}`))
    )

    await expect(startTransfer(store, planter, PLANTER, 101, 5)).rejects.toMatchObject({
      code: 'host_quota_exceeded',
    })
    // Witness: the re-check ran on the carried directories and failed.
    expect(listings.length).toBeGreaterThan(0)
    expect(probes.length).toBeGreaterThanOrEqual(ids.length)

    faults.clear()
    await store.cleanupExpired()
    const admitted = await startTransfer(store, planter, PLANTER, 102, 5)
    await store.fail(admitted.transfer.id, PLANTER)
  })

  // R3-F2, ported from the round-3 review's SOL-R3-HELD-ENOTDIR repro. The
  // owner's 40 bytes are adopted and evictable; an admission of 40 fits the
  // 80-byte budget beside them only if no phantom charge remains. A phantom
  // 40 bytes would make the admission evict every one of the owner's copies.
  it('SOL-R3-HELD-ENOTDIR: a carried charge below a caller root that became a file is dropped', async () => {
    const { store, owner, planter, ids } = await heldDuplicates()
    faults.clear()
    syncFs.rmSync(planter, { recursive: true, force: true })
    syncFs.writeFileSync(planter, '', { mode: 0o600 })
    // Witness: the held paths now answer ENOTDIR, not ENOENT.
    await expect(fs.lstat(downloadDirectory(planter, ids[0]!))).rejects.toMatchObject({
      code: 'ENOTDIR',
    })
    for (let sweep = 0; sweep < 3; sweep += 1) await store.cleanupExpired()

    const admitted = await startTransfer(store, owner, OWNER, 101, 40)
    for (const id of ids) expect(exists(downloadDirectory(owner, id))).toBe(true)
    await store.fail(admitted.transfer.id, OWNER)
  })

  it('a carried path that is no longer a directory is dropped while its parent cannot be listed', async () => {
    const { store, owner, planter, ids } = await heldDuplicates()
    faults.clear()
    const listings = failWithEio('readdir', target => target.endsWith(DOWNLOADS))
    for (const id of ids) {
      const target = downloadDirectory(planter, id)
      syncFs.rmSync(target, { recursive: true, force: true })
      syncFs.writeFileSync(target, '', { mode: 0o600 })
    }

    const admitted = await startTransfer(store, owner, OWNER, 101, 40)
    // Witness: the duplicates were unreachable, so only the lstat re-check
    // can have released their charge; no owner copy had to be evicted.
    expect(listings.length).toBeGreaterThan(0)
    for (const id of ids) expect(exists(downloadDirectory(owner, id))).toBe(true)
    await store.fail(admitted.transfer.id, OWNER)
  })

  it('a carried directory that is now indexed is charged once, not twice', async () => {
    const owner = callerDirectory(hostRoot, OWNER)
    const planter = callerDirectory(hostRoot, PLANTER)
    const ids = Array.from({ length: 8 }, () => randomUUID())
    plantCopies(owner, ids)
    const store = await openStore()
    // Only the planter's removals fail: the owner's indexed copies go.
    const renames = failRemovals(ids, `${path.sep}${PLANTER}${path.sep}`)
    plantCopies(planter, ids)

    const first = await store.cleanupExpired()
    expect(first.removeFailed).toBe(ids.length)
    expect(renames.length).toBeGreaterThanOrEqual(ids.length)
    for (const id of ids) expect(exists(downloadDirectory(owner, id))).toBe(false)

    // The next sweep finds each planter copy alone and indexes it as adopted.
    const second = await store.cleanupExpired()
    expect(second.removeFailed).toBe(0)
    expect((await store.debugInventory()).byCaller.get(PLANTER)).toEqual({ files: 8, bytes: 40 })

    // 40 indexed + 40 requested = 80 fits the budget exactly; a second charge
    // of the same 40 bytes would refuse it.
    const renamesBefore = renames.length
    const admitted = await startTransfer(store, planter, PLANTER, 101, 40)
    // Witness: nothing had to be evicted to make room.
    expect(renames.length).toBe(renamesBefore)
    await store.fail(admitted.transfer.id, PLANTER)
  })
})