Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
17.3 KB
Raw Blame
/**
 * On-disk and lifecycle states the GFS download store recovers from (PR #1028
 * review R1-M2, R1-L2, R1-L5, R1-L9): a store directory a shell made
 * unwritable, a non-regular `meta.json`, close() racing initialize() or
 * itself, and the removed-variable warning across initialize() retries.
 * statfs is the only boundary replaced, and only to size a small budget.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import * as net from 'node:net'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import {
  callerDirectory,
  completedCopy,
  digestOf,
  downloadDirectory,
  exists,
  freeSpaceSizedStatfs,
  metaFor,
  plantDownload,
  quotaCount,
  startTransfer,
} from '../__tests__/fixtures/gfsStoreTestKit'
import { logger } from '../logger'
import { GfsDownloadStore } from './gfsDownloadStore'
import { REMOVED_GFS_STORAGE_VARIABLES } from './gfsFilePolicy'

const { statfsBoundary, renameBoundary } = vi.hoisted(() => ({
  statfsBoundary: vi.fn(),
  renameBoundary: vi.fn(),
}))
vi.mock('node:fs/promises', async original => ({
  ...(await original<typeof fs>()),
  statfs: statfsBoundary,
  rename: renameBoundary,
}))

const A = 'caller-a'
const B = 'caller-b'
const C = 'caller-c'
const HOUR = 60 * 60_000
const REMOVED_VARIABLE_WARNING =
  'GFS download store ignores a removed retained-storage variable; the budget is MCP_HOST_GFS_DOWNLOAD_STORAGE_PERCENT of the workspace volume'

let nativeFs: typeof fs
let hostRoot: string
let outside: string
let rootA: string
let rootB: string
let rootC: string
const stores: GfsDownloadStore[] = []

/** statfs reports the real free space but a `totalBytes` volume (block size 1). */
function volumeOf(totalBytes: number): void {
  statfsBoundary.mockImplementation(async (target: string) => {
    const real = await nativeFs.statfs(target, { bigint: true })
    return { ...real, bsize: 1n, blocks: BigInt(totalBytes), bavail: real.bavail * real.bsize }
  })
}

function trackStore(root = hostRoot): GfsDownloadStore {
  const opened = new GfsDownloadStore(root)
  stores.push(opened)
  return opened
}

async function openStore(): Promise<GfsDownloadStore> {
  const opened = trackStore()
  await opened.initialize()
  return opened
}

/** Gives the owner rwx on every real directory under `root` so it can be removed. */
function restoreOwnerAccess(root: string): void {
  if (!exists(root) || !syncFs.lstatSync(root).isDirectory()) return
  syncFs.chmodSync(root, 0o700)
  for (const child of syncFs.readdirSync(root, { withFileTypes: true }))
    if (child.isDirectory()) restoreOwnerAccess(path.join(root, child.name))
}

/** The store's own directories, by name: input-<uuid> and .trash-<uuid>. */
function storeDirectories(callerRoot: string): string[] {
  const downloads = path.join(callerRoot, '.gfs-downloads')
  return exists(downloads) ? syncFs.readdirSync(downloads).sort() : []
}

beforeEach(async () => {
  nativeFs = await vi.importActual<typeof fs>('node:fs/promises')
  statfsBoundary.mockReset()
  statfsBoundary.mockImplementation(freeSpaceSizedStatfs(nativeFs.statfs))
  renameBoundary.mockReset()
  renameBoundary.mockImplementation((...args: unknown[]) =>
    Reflect.apply(nativeFs.rename, nativeFs, args)
  )
  hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-recovery-'))
  outside = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-recovery-outside-'))
  rootA = callerDirectory(hostRoot, A)
  rootB = callerDirectory(hostRoot, B)
  rootC = callerDirectory(hostRoot, C)
})

afterEach(async () => {
  vi.restoreAllMocks()
  vi.unstubAllEnvs()
  vi.useRealTimers()
  for (const opened of stores.splice(0)) await opened.close(0)
  restoreOwnerAccess(hostRoot)
  restoreOwnerAccess(outside)
  await nativeFs.rm(hostRoot, { recursive: true, force: true })
  await nativeFs.rm(outside, { recursive: true, force: true })
})

describe('GFS download store: a store directory made unwritable is still removed', () => {
  it('RCV-1: expired copies in input directories chmod 0500 are removed by the sweep, release their charge, and another caller is admitted', async () => {
    // Precondition: a 0500 directory refuses the removal of its children for
    // this user (it does not for root, which bypasses the mode).
    const probe = syncFs.mkdtempSync(path.join(outside, 'probe-'))
    syncFs.writeFileSync(path.join(probe, 'child'), 'x')
    syncFs.chmodSync(probe, 0o500)
    await expect(nativeFs.rm(probe, { recursive: true, force: true })).rejects.toMatchObject({
      code: 'EACCES',
    })
    syncFs.chmodSync(probe, 0o700)

    vi.useFakeTimers({ toFake: ['Date'] })
    vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))
    // 100-byte volume, budget 70: A and B hold 30 bytes each.
    volumeOf(100)
    const store = await openStore()
    const copyA = await completedCopy(store, rootA, A, 800, 30)
    const copyB = await completedCopy(store, rootB, B, 801, 30)
    const directoryA = downloadDirectory(rootA, copyA.receipt.id)
    const directoryB = downloadDirectory(rootB, copyB.receipt.id)
    // A symlink inside the tree to a directory outside the Host root whose
    // mode must not change: the walk never follows it.
    const target = path.join(outside, 'target')
    syncFs.mkdirSync(target, { mode: 0o500 })
    syncFs.symlinkSync(target, path.join(directoryA, 'link'))
    syncFs.chmodSync(directoryA, 0o500)
    syncFs.chmodSync(directoryB, 0o500)
    vi.setSystemTime(Date.now() + 2 * HOUR)
    const budgetDenied = await quotaCount('host', 'storage_bytes')

    const sweep = await store.cleanupExpired()

    expect(sweep).toMatchObject({ removedExpired: 2, removeFailed: 0 })
    expect(storeDirectories(rootA)).toEqual([])
    expect(storeDirectories(rootB)).toEqual([])
    const admitted = await startTransfer(store, rootC, C, 802, 30)
    expect(admitted.transfer.sizeBytes).toBe(30)
    expect(await quotaCount('host', 'storage_bytes')).toBe(budgetDenied)
    // The symlink was removed, never followed: its target keeps its mode.
    expect(syncFs.statSync(target).mode & 0o777).toBe(0o500)
  })

  /** A valid planted copy of `sizeBytes` zero bytes; the source is sparse. */
  function plantSparseCopy(callerRoot: string, sizeBytes: number) {
    const id = randomUUID()
    const bytes = Buffer.alloc(sizeBytes)
    const planted = plantDownload(callerRoot, {
      id,
      bytes: Buffer.alloc(0),
      meta: metaFor(id, bytes, Date.now()),
    })
    syncFs.truncateSync(path.join(planted.directory, 'source'), sizeBytes)
    return planted
  }

  /** Whether `candidate` names `directory` in its .gfs-downloads parent. */
  function isPlantedPath(candidate: string, directory: string): boolean {
    return (
      path.basename(candidate) === path.basename(directory) &&
      path.basename(path.dirname(candidate)) === '.gfs-downloads'
    )
  }

  // Root bypasses the mode bits, so the refused rename cannot be produced.
  it.skipIf(process.getuid?.() === 0)(
    'RCV-1c: an adopted copy whose .gfs-downloads parent is chmod 0500 is evicted after the owner rwx is restored on the parent, and its charge is released',
    async () => {
      // 100-byte volume, budget 70: the adopted 50 bytes plus B's 30 exceed it.
      volumeOf(100)
      const planted = plantSparseCopy(rootA, 50)
      const parent = path.dirname(planted.directory)
      const store = await openStore()
      syncFs.chmodSync(parent, 0o500)
      // Precondition: the parent refuses a rename out of it for this user.
      const probe = path.join(parent, `probe-${randomUUID()}`)
      expect(() => syncFs.renameSync(planted.directory, probe)).toThrow(
        expect.objectContaining({ code: 'EACCES' })
      )
      const budgetDenied = await quotaCount('host', 'storage_bytes')

      const admitted = await startTransfer(store, rootB, B, 803, 30)

      expect(admitted.transfer.sizeBytes).toBe(30)
      expect(exists(planted.directory)).toBe(false)
      expect(storeDirectories(rootA)).toEqual([])
      expect(await quotaCount('host', 'storage_bytes')).toBe(budgetDenied)
      // Witness: the first rename was refused and the retry succeeded on the
      // parent whose owner bits were restored, other bits kept.
      // Paths are matched by name: the store works on resolved paths.
      const renamesOutOfParent = renameBoundary.mock.calls.filter(call =>
        isPlantedPath(String(call[0]), planted.directory)
      )
      expect(renamesOutOfParent).toHaveLength(2)
      expect(syncFs.statSync(parent).mode & 0o777).toBe(0o700)
    }
  )

  it.skipIf(process.getuid?.() === 0)(
    'RCV-1d: a symlink swapped in for the .gfs-downloads parent after a refused rename is not followed, and the removal is refused',
    async () => {
      vi.useFakeTimers({ toFake: ['Date'] })
      vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))
      volumeOf(100)
      const planted = plantSparseCopy(rootA, 20)
      const parent = path.dirname(planted.directory)
      const store = await openStore()
      // The directory the symlink will lead to; its mode must not change.
      const target = path.join(outside, 'target')
      syncFs.mkdirSync(target, { mode: 0o500 })
      const movedParent = path.join(outside, 'moved-downloads')
      let swapped = 0
      renameBoundary.mockImplementation(async (...args: unknown[]) => {
        const from = String(args[0])
        if (swapped === 0 && isPlantedPath(from, planted.directory)) {
          swapped += 1
          // The refused rename, then the parent swapped for a symlink before
          // the store restores the parent's owner access.
          await nativeFs.rename(path.dirname(from), movedParent)
          await nativeFs.symlink(target, path.dirname(from))
          throw Object.assign(new Error('EACCES: injected'), { code: 'EACCES' })
        }
        return Reflect.apply(nativeFs.rename, nativeFs, args)
      })
      vi.setSystemTime(Date.now() + 2 * HOUR)

      const sweep = await store.cleanupExpired()

      expect(swapped).toBe(1)
      expect(sweep).toMatchObject({ removedExpired: 0, removeFailed: 1 })
      // The symlink was refused, never followed: its target keeps its mode.
      expect(syncFs.lstatSync(parent).isSymbolicLink()).toBe(true)
      expect(syncFs.statSync(target).mode & 0o777).toBe(0o500)
      expect(exists(path.join(movedParent, path.basename(planted.directory)))).toBe(true)
    }
  )
})

describe('GFS download store: a non-regular meta.json', () => {
  it('RCV-2: a Unix socket planted as meta.json makes the directory incomplete, and the sweep removes it', async () => {
    const store = await openStore()
    const kept = await completedCopy(store, rootA, A, 810, 8)
    const id = randomUUID()
    const planted = plantDownload(rootA, {
      id,
      bytes: Buffer.alloc(8, 1),
      meta: 'omit',
    })
    // A socket path is limited to about 100 bytes, so it is bound at a short
    // path and renamed into place; the server is closed once it is there.
    const socketPath = path.join(outside, 's')
    const server = net.createServer()
    await new Promise<void>(resolve => server.listen(socketPath, resolve))
    syncFs.renameSync(socketPath, path.join(planted.directory, 'meta.json'))
    await new Promise<void>(resolve => server.close(() => resolve()))
    expect(syncFs.lstatSync(path.join(planted.directory, 'meta.json')).isSocket()).toBe(true)

    const sweep = await store.cleanupExpired()

    expect(sweep.removedIncomplete).toBe(1)
    expect(exists(planted.directory)).toBe(false)
    // Witness: the sweep inspected the tree and kept the valid sibling.
    expect(exists(downloadDirectory(rootA, kept.receipt.id))).toBe(true)
    await expect(store.readManagedFile(kept.receipt.path, A)).resolves.toEqual(kept.bytes)
  })

  it('RCV-2b: at initialize, a socket meta.json is removed as incomplete', async () => {
    const id = randomUUID()
    const bytes = Buffer.alloc(8, 2)
    const validId = randomUUID()
    const valid = plantDownload(rootB, {
      id: validId,
      bytes,
      meta: metaFor(validId, bytes, Date.now()),
    })
    const planted = plantDownload(rootA, { id, bytes, meta: 'omit' })
    const socketPath = path.join(outside, 's')
    const server = net.createServer()
    await new Promise<void>(resolve => server.listen(socketPath, resolve))
    syncFs.renameSync(socketPath, path.join(planted.directory, 'meta.json'))
    await new Promise<void>(resolve => server.close(() => resolve()))
    const info = vi.spyOn(logger, 'info')

    await openStore()

    expect(exists(planted.directory)).toBe(false)
    expect(info).toHaveBeenCalledWith(
      expect.objectContaining({ component: 'GfsDownloadStore', removedIncomplete: 1, adopted: 1 }),
      'GFS download store initialized'
    )
    // Witness: the sweep inspected the tree and adopted the valid sibling.
    expect(exists(valid.directory)).toBe(true)
  })
})

describe('GFS download store: close() and initialize()', () => {
  function blockSweeps(target: GfsDownloadStore) {
    const internals = target as unknown as { sweep: (now: number) => Promise<unknown> }
    const realSweep = internals.sweep.bind(target)
    const started: number[] = []
    const releases: Array<() => void> = []
    const finished: number[] = []
    vi.spyOn(internals, 'sweep').mockImplementation(async (now: number) => {
      const index = started.length
      started.push(index)
      await new Promise<void>(resolve => releases.push(resolve))
      const result = await realSweep(now)
      finished.push(index)
      return result
    })
    return { started, releases, finished }
  }

  it('RCV-3: close() requested while initialize() is sweeping waits for that sweep', async () => {
    const store = trackStore()
    const sweeps = blockSweeps(store)
    const initializing = store.initialize()
    const initialized = initializing.catch((error: unknown) => error)
    await vi.waitFor(() => expect(sweeps.started).toEqual([0]))
    let closed = false
    const closing = store.close(5_000).then(() => {
      closed = true
    })

    await new Promise(resolve => setTimeout(resolve, 20))
    expect(closed).toBe(false)

    sweeps.releases[0]!()
    await closing
    // Witness: the initialization sweep finished before close() resolved.
    expect(sweeps.finished).toEqual([0])
    expect(closed).toBe(true)
    await expect(initialized).resolves.toMatchObject({ code: 'download_busy' })
    expect(store.isAvailable()).toBe(false)
  })

  it('RCV-4: a second close() while the first is draining resolves only when the first does', async () => {
    const store = await openStore()
    const { transfer, bytes } = await startTransfer(store, rootA, A, 820, 8)
    let firstClosed = false
    let secondClosed = false
    const first = store.close(5_000).then(() => {
      firstClosed = true
    })
    const second = store.close(5_000).then(() => {
      secondClosed = true
    })

    await new Promise(resolve => setTimeout(resolve, 20))
    expect(firstClosed).toBe(false)
    expect(secondClosed).toBe(false)

    // Witness: the transfer close() waits for completes while it drains.
    const receipt = await store.publish(transfer.id, A, digestOf(bytes))
    expect(receipt.id).toBe(transfer.id)
    await Promise.all([first, second])
    expect(firstClosed).toBe(true)
    expect(secondClosed).toBe(true)
  })

  it('RCV-5: a managed read whose digest check finishes after close() does not remove the copy', async () => {
    const store = await openStore()
    const copy = await completedCopy(store, rootA, A, 830, 8)
    const directory = downloadDirectory(rootA, copy.receipt.id)
    // Same size, mode and inode, other bytes: only the digest check refuses it.
    syncFs.writeFileSync(path.join(directory, 'source'), Buffer.alloc(8, 0xee))
    const internals = store as unknown as { openEntryContent: (entry: unknown) => Promise<unknown> }
    const realOpen = internals.openEntryContent.bind(store)
    let releaseOpen!: () => void
    const opened: unknown[] = []
    vi.spyOn(internals, 'openEntryContent').mockImplementation(async (entry: unknown) => {
      await new Promise<void>(resolve => {
        releaseOpen = resolve
      })
      const handle = await realOpen(entry)
      opened.push(handle)
      return handle
    })

    const read = store.readManagedFile(copy.receipt.path, A).catch((error: unknown) => error)
    await vi.waitFor(() => expect(releaseOpen).toBeDefined())
    await store.close(5_000)
    releaseOpen()

    await expect(read).resolves.toMatchObject({ code: 'download_missing' })
    // Witness: the read opened the copy and reached the digest check.
    expect(opened).toHaveLength(1)
    expect(exists(directory)).toBe(true)
  })
})

describe('GFS download store: removed variables across initialize() retries', () => {
  it('RCV-6: a removed variable is warned about once although initialize() runs twice', async () => {
    for (const name of REMOVED_GFS_STORAGE_VARIABLES) vi.stubEnv(name, undefined)
    vi.stubEnv(REMOVED_GFS_STORAGE_VARIABLES[0]!, '1')
    // The Host root is a regular file first, so the first attempt fails.
    const root = path.join(outside, 'host-root')
    syncFs.writeFileSync(root, 'not a directory')
    const store = trackStore(root)
    const warn = vi.spyOn(logger, 'warn')

    await expect(store.initialize()).rejects.toMatchObject({ code: 'workspace_unavailable' })
    syncFs.rmSync(root)
    await store.initialize()

    // Witness: both attempts ran; the second one initialized the store.
    expect(store.isAvailable()).toBe(true)
    expect(warn.mock.calls.filter(call => call[1] === REMOVED_VARIABLE_WARNING)).toHaveLength(1)
  })
})