Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
16.7 KB
Raw Blame
/**
 * Restart behaviour of the filesystem GFS download store: a volume written by
 * the ledger store of the pre-#1028 image (snapshots generated by running dev
 * 74e0d81d9's store, see devGfsStoreFixture)
 * never blocks admission, and a volume written by this store is swept but its
 * copies are never reused or served after a restart.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import { type DevSnapshot, loadDevStore } from '../__tests__/fixtures/devGfsStoreFixture'
import {
  callerDirectory,
  completedCopy,
  downloadDirectory,
  exists,
  expiryCount,
  sourceFor,
} from '../__tests__/fixtures/gfsStoreTestKit'
import { logger } from '../logger'
import { isProtectedWorkspacePath } from '../workspace/protectedPaths'
import { GfsDownloadStore } from './gfsDownloadStore'

const { renameFailure, rmFailure } = vi.hoisted(() => ({
  /** `target` is a Host-root entry name; the store renames it by its real path. */
  renameFailure: { target: undefined as string | undefined, injected: 0 },
  /** `prefix` is a Host-root entry name prefix the next `rm` refuses once. */
  rmFailure: { prefix: undefined as string | undefined, injected: 0 },
}))
// Pass-through `rename` and `rm`: R3 makes the retirement rename of the legacy
// store fail once and R5 the removal of the retired tree, as a busy mount
// would (EBUSY: EACCES and EPERM from rm restore owner access and retry once).
vi.mock('node:fs/promises', async original => {
  const actual = await original<typeof import('node:fs/promises')>()
  const { freeSpaceSizedStatfs } = await import('../__tests__/fixtures/gfsStoreTestKit')
  return {
    ...actual,
    // The volume sized to its free space, so the disk's occupancy never
    // meets the store's free-space floor.
    statfs: freeSpaceSizedStatfs(actual.statfs),
    rename: async (...values: Parameters<typeof actual.rename>) => {
      if (
        renameFailure.target !== undefined &&
        String(values[0]).endsWith(`${path.sep}${renameFailure.target}`)
      ) {
        renameFailure.target = undefined
        renameFailure.injected += 1
        throw Object.assign(new Error('simulated rename failure'), { code: 'EACCES' })
      }
      return actual.rename(...values)
    },
    rm: async (...values: Parameters<typeof actual.rm>) => {
      if (
        rmFailure.prefix !== undefined &&
        path.basename(String(values[0])).startsWith(rmFailure.prefix)
      ) {
        rmFailure.prefix = undefined
        rmFailure.injected += 1
        throw Object.assign(new Error('simulated rm failure'), { code: 'EBUSY' })
      }
      return actual.rm(...values)
    },
  }
})

const CALLER = 'caller-a'
const RETIRED_PREFIX = '.gfs-download-store.retired-'
const RETIREMENT_MESSAGE = 'Retired the pre-#1028 GFS download store'
const INPUT_RE = /^input-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/

let hostRoot: string
const stores: GfsDownloadStore[] = []

async function open(): Promise<GfsDownloadStore> {
  const store = new GfsDownloadStore(hostRoot)
  stores.push(store)
  await store.initialize()
  return store
}

type LogCall = [Record<string, unknown>, string]

function startupLines(info: { mock: { calls: unknown[][] } }): Record<string, unknown>[] {
  return (info.mock.calls as LogCall[])
    .filter(([, message]) => message === 'GFS download store initialized')
    .map(([fields]) => fields)
}

function lastStartupLine(info: { mock: { calls: unknown[][] } }): Record<string, unknown> {
  const lines = startupLines(info)
  if (lines.length === 0) throw new Error('the store logged no startup line')
  return lines[lines.length - 1]!
}

function retirementWarnings(warn: { mock: { calls: unknown[][] } }): Record<string, unknown>[] {
  return (warn.mock.calls as LogCall[])
    .filter(([, message]) => message === RETIREMENT_MESSAGE)
    .map(([fields]) => fields)
}

async function hostRootEntries(): Promise<string[]> {
  return (await fs.readdir(hostRoot)).sort()
}

beforeEach(async () => {
  hostRoot = await fs.realpath(await fs.mkdtemp(path.join(tmpdir(), 'gfs-store-restart-')))
})

afterEach(async () => {
  renameFailure.target = undefined
  renameFailure.injected = 0
  rmFailure.prefix = undefined
  rmFailure.injected = 0
  for (const store of stores.splice(0)) await store.close()
  vi.restoreAllMocks()
  await fs.rm(hostRoot, { recursive: true, force: true })
})

describe('GFS download store restart', () => {
  it.each([
    ['killed with an expired and a live processing lease', 'crashed', undefined],
    ['restarted by the dev Host, every record quarantined (#1019)', 'restarted', false],
  ] as Array<[string, DevSnapshot, boolean | undefined]>)(
    'R1: restarting on a volume written by the dev store never blocks admission: %s',
    async (_label, snapshot, oldStoreAvailable) => {
      const dev = loadDevStore(hostRoot, snapshot)
      // Witness: the old store, both of its leases and its copies are on disk
      // before the start; on `restarted` the dev store itself was unavailable.
      expect(exists(dev.storeRoot)).toBe(true)
      expect(Date.parse(dev.leases.live.expiresAt)).toBeGreaterThan(Date.now())
      expect(Date.parse(dev.leases.expired.expiresAt)).toBeLessThan(Date.now())
      expect(dev.userDirectories.length).toBeGreaterThan(0)
      expect(dev.oldStoreAvailable).toBe(oldStoreAvailable)
      const warn = vi.spyOn(logger, 'warn')
      const info = vi.spyOn(logger, 'info')
      const retired = await expiryCount('retired_legacy_store')

      const store = await open()

      expect(store.isAvailable()).toBe(true)
      const retirements = retirementWarnings(warn)
      expect(retirements).toHaveLength(1)
      expect(retirements[0]).toMatchObject({ component: 'GfsDownloadStore' })
      expect(retirements[0]!.files as number).toBeGreaterThan(0)
      expect(retirements[0]!.bytes as number).toBeGreaterThan(0)
      expect(lastStartupLine(info)).toMatchObject({
        retiredLegacyStore: true,
        removedIncomplete: dev.userDirectories.length,
      })
      expect(await expiryCount('retired_legacy_store')).toBe(retired + 1)
      expect(exists(dev.storeRoot)).toBe(false)
      expect((await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))).toEqual([])
      const { receipt, bytes } = await completedCopy(
        store,
        dev.callerRoot,
        dev.callerIdentity,
        1,
        64
      )
      await expect(store.readManagedFile(receipt.path, dev.callerIdentity)).resolves.toEqual(bytes)
    }
  )

  it('R2: dev user directories of every state are removed at startup and do not count against quota', async () => {
    const dev = loadDevStore(hostRoot, 'crashed')
    // Witness: every dev directory and its bytes are on disk before the start,
    // with a completed, a missing and a transferring record among them.
    expect(new Set(Object.values(dev.recordStates))).toEqual(
      new Set(['completed', 'missing', 'transferring'])
    )
    expect(dev.userDirectories).toHaveLength(4)
    expect(dev.userBytes).toBeGreaterThan(0)
    for (const directory of dev.userDirectories) expect(exists(directory)).toBe(true)
    const info = vi.spyOn(logger, 'info')

    const store = await open()

    expect(lastStartupLine(info)).toMatchObject({
      removedIncomplete: 4,
      retainedCompleted: 0,
      retainedBytes: 0,
      adopted: 0,
    })
    for (const directory of dev.userDirectories) expect(exists(directory)).toBe(false)
    expect(await store.debugInventory()).toMatchObject({ bytes: 0, files: 0 })
    const { bytes } = await completedCopy(store, dev.callerRoot, dev.callerIdentity, 2, 64)
    expect(await store.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
  })

  it('R3: a failed retirement is logged, the store is available and the next sweep retires it', async () => {
    const dev = loadDevStore(hostRoot, 'restarted')
    const warn = vi.spyOn(logger, 'warn')
    const removeFailed = await expiryCount('remove_failed')
    renameFailure.target = '.gfs-download-store'

    const store = await open()

    // Witness: the retirement rename was attempted and failed exactly once.
    expect(renameFailure.injected).toBe(1)
    expect(warn).toHaveBeenCalledWith(
      { component: 'GfsDownloadStore', code: 'EACCES', step: 'rename' },
      expect.stringContaining('could not retire')
    )
    expect(await expiryCount('remove_failed')).toBe(removeFailed + 1)
    expect(store.isAvailable()).toBe(true)
    expect(retirementWarnings(warn)).toEqual([])
    expect(exists(dev.storeRoot)).toBe(true)
    const root = callerDirectory(hostRoot, CALLER)
    const { receipt, bytes } = await completedCopy(store, root, CALLER, 3, 64)
    await expect(store.readManagedFile(receipt.path, CALLER)).resolves.toEqual(bytes)

    // The periodic sweep retries without a restart.
    await expect(store.cleanupExpired()).resolves.toMatchObject({ removeFailed: 0 })

    expect(retirementWarnings(warn)).toHaveLength(1)
    expect(exists(dev.storeRoot)).toBe(false)
    expect(await hostRootEntries()).toEqual(['users'])
  })

  it('R4: after startup the volume has no store directory and only valid input directories', async () => {
    const dev = loadDevStore(hostRoot, 'restarted')
    const store = await open()
    const published = await completedCopy(store, dev.callerRoot, dev.callerIdentity, 7, 64)
    await completedCopy(store, callerDirectory(hostRoot, CALLER), CALLER, 8, 64)

    expect(await hostRootEntries()).toEqual(['users'])
    let directories = 0
    for (const user of await fs.readdir(path.join(hostRoot, 'users'))) {
      const downloads = path.join(hostRoot, 'users', user, '.gfs-downloads')
      if (!exists(downloads)) continue
      for (const name of await fs.readdir(downloads)) {
        directories += 1
        expect(name).toMatch(INPUT_RE)
        const directory = path.join(downloads, name)
        expect((await fs.readdir(directory)).sort()).toEqual(['meta.json', 'source'])
        const meta = JSON.parse(await fs.readFile(path.join(directory, 'meta.json'), 'utf8'))
        expect(meta).toMatchObject({ schemaVersion: 1, id: name.slice('input-'.length) })
        const source = await fs.lstat(path.join(directory, 'source'))
        expect(source.isFile()).toBe(true)
        expect(source.size).toBe(meta.sizeBytes)
      }
    }
    // Witness: the predicate walked both published downloads.
    expect(directories).toBe(2)
    expect(exists(downloadDirectory(dev.callerRoot, published.receipt.id))).toBe(true)
  })

  it('R5: a retired directory left by a failed removal is removed by the next sweep and is protected meanwhile', async () => {
    const dev = loadDevStore(hostRoot, 'restarted')
    const warn = vi.spyOn(logger, 'warn')
    const removeFailed = await expiryCount('remove_failed')
    rmFailure.prefix = RETIRED_PREFIX

    const store = await open()

    // Witness: the old store was renamed and the removal of the renamed tree failed once.
    expect(rmFailure.injected).toBe(1)
    expect(retirementWarnings(warn)).toHaveLength(1)
    expect(warn).toHaveBeenCalledWith(
      { component: 'GfsDownloadStore', code: 'EBUSY', step: 'remove' },
      expect.stringContaining('could not retire')
    )
    expect(await expiryCount('remove_failed')).toBe(removeFailed + 1)
    expect(store.isAvailable()).toBe(true)
    expect(exists(dev.storeRoot)).toBe(false)
    const leftovers = (await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))
    expect(leftovers).toHaveLength(1)
    expect(isProtectedWorkspacePath(`${leftovers[0]}/ledger-v1.json`)).toBe(true)

    await expect(store.cleanupExpired()).resolves.toMatchObject({ removeFailed: 0 })

    expect(await hostRootEntries()).toEqual(['users'])
  })

  it('R6: a retired directory left by an earlier start is removed at startup and is protected meanwhile', async () => {
    const leftover = `${RETIRED_PREFIX}00000000-0000-4000-8000-000000000000`
    await fs.mkdir(path.join(hostRoot, leftover), { mode: 0o700 })
    await fs.writeFile(path.join(hostRoot, leftover, 'ledger-v1.json'), '{}', { mode: 0o600 })
    expect(isProtectedWorkspacePath(`${leftover}/ledger-v1.json`)).toBe(true)
    expect(isProtectedWorkspacePath(leftover)).toBe(true)
    // Witness: the check is a prefix match, not a blanket refusal.
    expect(isProtectedWorkspacePath('.gfs-download-storex/ledger-v1.json')).toBe(false)
    const info = vi.spyOn(logger, 'info')
    const removeFailed = await expiryCount('remove_failed')

    const store = await open()

    expect(store.isAvailable()).toBe(true)
    // Nothing named .gfs-download-store was found, so nothing was retired this start.
    expect(lastStartupLine(info)).toMatchObject({ retiredLegacyStore: false })
    expect(await expiryCount('remove_failed')).toBe(removeFailed)
    expect(exists(path.join(hostRoot, leftover))).toBe(false)
    expect(await hostRootEntries()).toEqual([])
  })

  it('R7: restarting with copies left by this design keeps them for accounting and does not reuse them', async () => {
    const root = callerDirectory(hostRoot, CALLER)
    const first = await open()
    const { receipt, bytes } = await completedCopy(first, root, CALLER, 9, 64)
    // Witness: before the restart the copy is reused and served.
    await expect(first.reusableReceipt(CALLER, sourceFor(9), bytes.byteLength)).resolves.toEqual(
      receipt
    )
    await first.close()
    const info = vi.spyOn(logger, 'info')

    const second = await open()

    expect(lastStartupLine(info)).toMatchObject({
      retiredLegacyStore: false,
      removedIncomplete: 0,
      retainedCompleted: 1,
      retainedBytes: bytes.byteLength,
      adopted: 1,
    })
    expect(await second.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
    await expect(
      second.reusableReceipt(CALLER, sourceFor(9), bytes.byteLength)
    ).resolves.toBeUndefined()
    await expect(second.readManagedFile(receipt.path, CALLER)).rejects.toMatchObject({
      code: 'download_missing',
    })
    expect(exists(downloadDirectory(root, receipt.id))).toBe(true)
  })

  it('R8: after a restart the same file downloads again under a new id that is reused and served', async () => {
    const root = callerDirectory(hostRoot, CALLER)
    const first = await open()
    const before = await completedCopy(first, root, CALLER, 10, 64)
    await first.close()

    const second = await open()
    const after = await completedCopy(second, root, CALLER, 10, 64)

    expect(after.receipt.id).not.toBe(before.receipt.id)
    await expect(second.readManagedFile(after.receipt.path, CALLER)).resolves.toEqual(after.bytes)
    await expect(second.reusableReceipt(CALLER, sourceFor(10), 64)).resolves.toEqual(after.receipt)
    // The adopted copy still counts until it expires or is evicted.
    expect(await second.debugInventory()).toMatchObject({ bytes: 128, files: 2 })
  })
})

describe('GFS download store restart: retirement edge cases', () => {
  it('S14: only `.gfs-download-store.retired-<uuid>` names are removed', async () => {
    const foreign = path.join(hostRoot, `${RETIRED_PREFIX}foo`)
    const retired = path.join(hostRoot, `${RETIRED_PREFIX}00000000-0000-4000-8000-000000000001`)
    for (const directory of [foreign, retired]) {
      await fs.mkdir(directory, { mode: 0o700 })
      await fs.writeFile(path.join(directory, 'copy'), 'bytes', { mode: 0o600 })
    }

    await open()

    // Witness: a uuid-named retired tree is removed by the same sweep.
    expect(exists(retired)).toBe(false)
    expect(exists(path.join(foreign, 'copy'))).toBe(true)
  })

  it.each([
    ['a file', 'file'],
    ['a symlink', 'symlink'],
  ] as const)(
    'S15: a `.gfs-download-store` that is %s is retired without following it',
    async (_label, kind) => {
      const legacy = path.join(hostRoot, '.gfs-download-store')
      const outside = await fs.mkdtemp(path.join(tmpdir(), 'gfs-store-legacy-target-'))
      const target = path.join(outside, 'target')
      try {
        await fs.writeFile(target, 'outside bytes', { mode: 0o600 })
        if (kind === 'file') await fs.writeFile(legacy, 'legacy bytes', { mode: 0o600 })
        else await fs.symlink(target, legacy)
        const warn = vi.spyOn(logger, 'warn')
        const retiredBefore = await expiryCount('retired_legacy_store')

        const store = await open()

        expect(store.isAvailable()).toBe(true)
        expect(retirementWarnings(warn)).toEqual([
          expect.objectContaining({ files: kind === 'file' ? 1 : 0 }),
        ])
        expect(await expiryCount('retired_legacy_store')).toBe(retiredBefore + 1)
        expect(exists(legacy)).toBe(false)
        expect((await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))).toEqual(
          []
        )
        // The symlink's target is never followed.
        await expect(fs.readFile(target, 'utf8')).resolves.toBe('outside bytes')
      } finally {
        await fs.rm(outside, { recursive: true, force: true })
      }
    }
  )
})