Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
10.4 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { register } from 'prom-client'
import { type GfsRuntime, bootstrapGfsRuntime } from '../gfsRuntime'
import { GfsDownloadStore } from '../internalTools/gfsDownloadStore'
import { logger } from '../logger'

// statfs is the only boundary replaced: the volume sized to its free space,
// so the disk's occupancy never meets the store's free-space floor.
vi.mock('node:fs/promises', async original => {
  const actual = await original<typeof fs>()
  const { freeSpaceSizedStatfs } = await import('./fixtures/gfsStoreTestKit')
  return { ...actual, statfs: freeSpaceSizedStatfs(actual.statfs) }
})

const roots: string[] = []
const runtimes: GfsRuntime[] = []

function deferred() {
  let resolve!: () => void
  const promise = new Promise<void>(done => {
    resolve = done
  })
  return { promise, resolve }
}

async function root() {
  const value = await fs.mkdtemp(join(tmpdir(), 'gfs-runtime-'))
  roots.push(value)
  return value
}

async function start(hostRoot: string, cycleMs?: number): Promise<GfsRuntime> {
  const runtime = await bootstrapGfsRuntime(hostRoot, cycleMs === undefined ? {} : { cycleMs })
  runtimes.push(runtime)
  return runtime
}

/** A Host root that is a symlink: initialize refuses it with workspace_unavailable. */
async function symlinkedHostRoot(): Promise<{ link: string; target: string }> {
  const base = await root()
  const target = join(base, 'real-host')
  await fs.mkdir(target, { mode: 0o700 })
  const link = join(base, 'host-link')
  await fs.symlink(target, link)
  return { link, target }
}

async function expiryCount(outcome: string): Promise<number> {
  const metric = register.getSingleMetric('clerum_gfs_download_expiry_total')
  if (metric === undefined) throw new Error('expiry metric is not registered')
  const { values } = await metric.get()
  return values
    .filter(sample => sample.labels.outcome === outcome)
    .reduce((sum, sample) => sum + sample.value, 0)
}

afterEach(async () => {
  vi.useRealTimers()
  for (const runtime of runtimes.splice(0)) await runtime.stop()
  vi.restoreAllMocks()
  for (const value of roots.splice(0)) await fs.rm(value, { recursive: true, force: true })
})

describe('GFS runtime bootstrap', () => {
  it('bootstraps an available store and a workspace provider', async () => {
    const runtime = await start(await root())
    expect(runtime.store.isAvailable()).toBe(true)
    expect(runtime.workspaceProvider).toBeDefined()
  })

  it('rejects a cycle interval below 25ms', async () => {
    await expect(bootstrapGfsRuntime(await root(), { cycleMs: 24 })).rejects.toThrow(
      'GFS runtime cycleMs must be an integer of at least 25ms'
    )
  })

  it('a failed initialize is logged with its code and the next cycle makes the store available', async () => {
    const { link } = await symlinkedHostRoot()
    const error = vi.spyOn(logger, 'error')
    const initialize = vi.spyOn(GfsDownloadStore.prototype, 'initialize')

    const runtime = await start(link, 25)

    expect(runtime.store.isAvailable()).toBe(false)
    expect(error).toHaveBeenCalledWith(
      { component: 'gfs-runtime', code: 'workspace_unavailable' },
      expect.stringContaining('retried by the next cycle')
    )
    // The cause goes away: the symlink becomes a real directory.
    await fs.rm(link)
    await fs.mkdir(link, { mode: 0o700 })
    await vi.waitFor(() => expect(runtime.store.isAvailable()).toBe(true), {
      timeout: 5_000,
      interval: 10,
    })
    // Witness: availability came from a cycle-driven initialize, not the first one.
    expect(initialize.mock.calls.length).toBeGreaterThanOrEqual(2)
  })

  it('the cycle sweeps expired downloads', async () => {
    const hostRoot = await root()
    const callerRoot = join(hostRoot, 'users', 'caller-a')
    await fs.mkdir(callerRoot, { recursive: true, mode: 0o700 })
    vi.useFakeTimers({ toFake: ['Date'] })
    vi.setSystemTime(new Date('2026-10-08T10:00:00.000Z'))
    const runtime = await start(hostRoot, 25)
    const bytes = Buffer.from('expired content')
    const transfer = await runtime.store.createTransfer({
      callerIdentity: 'caller-a',
      callerWorkspacePath: callerRoot,
      source: {
        kind: 'gfs',
        drive: 'main',
        resourceId: 'f'.repeat(32),
        gfsUri: `gfs://main/${'f'.repeat(32)}`,
        name: 'expired.bin',
        version: 1,
      },
      sizeBytes: bytes.byteLength,
      expiresAt: new Date(Date.now() + 60_000).toISOString(),
    })
    await fs.writeFile(join(callerRoot, transfer.partialPath), bytes)
    await runtime.store.publish(
      transfer.id,
      'caller-a',
      createHash('sha256').update(bytes).digest('hex')
    )
    const directory = join(callerRoot, '.gfs-downloads', `input-${transfer.id}`)
    // Witness: the published download is on disk before it expires.
    await expect(fs.lstat(join(directory, 'source'))).resolves.toBeDefined()
    const removed = await expiryCount('expired_removed')

    vi.setSystemTime(Date.now() + 120_000)

    // The name disappears at the rename to `.trash-<uuid>`; the removal is
    // counted only after the rm is proven, so both are awaited together.
    await vi.waitFor(
      async () => {
        await expect(fs.lstat(directory)).rejects.toMatchObject({ code: 'ENOENT' })
        expect(await expiryCount('expired_removed')).toBe(removed + 1)
      },
      { timeout: 5_000, interval: 10 }
    )
  })

  it('a cleanup that throws is logged and the next cycle sweeps again', async () => {
    const runtime = await start(await root(), 25)
    const warn = vi.spyOn(logger, 'warn')
    const cleanup = vi
      .spyOn(GfsDownloadStore.prototype, 'cleanupExpired')
      .mockImplementationOnce(async () => {
        throw Object.assign(new Error('injected EIO'), { code: 'EIO' })
      })

    // Witness: a second cycle reached cleanup after the first one threw.
    await vi.waitFor(() => expect(cleanup.mock.calls.length).toBeGreaterThanOrEqual(2), {
      timeout: 5_000,
      interval: 10,
    })

    expect(warn).toHaveBeenCalledWith(
      { component: 'gfs-runtime', code: 'EIO' },
      'GFS download cleanup failed; the next cycle retries it'
    )
    expect(runtime.store.isAvailable()).toBe(true)
  })

  it('stop joins the cycle in flight and closes the store', async () => {
    const runtime = await start(await root(), 25)
    const sweepStarted = deferred()
    const releaseSweep = deferred()
    const cleanup = vi
      .spyOn(GfsDownloadStore.prototype, 'cleanupExpired')
      .mockImplementationOnce(async () => {
        sweepStarted.resolve()
        await releaseSweep.promise
        return { removedExpired: 0, removedIncomplete: 0, removeFailed: 0 }
      })
    await sweepStarted.promise
    let stopped = false
    const stopping = runtime.stop().then(() => {
      stopped = true
    })
    await new Promise(resolve => setTimeout(resolve, 50))
    expect(stopped).toBe(false)

    releaseSweep.resolve()
    await stopping

    expect(cleanup).toHaveBeenCalledOnce()
    expect(runtime.store.isAvailable()).toBe(false)
    await expect(
      runtime.store.createTransfer({
        callerIdentity: 'caller-a',
        callerWorkspacePath: join(tmpdir(), 'unused'),
        source: {
          kind: 'gfs',
          drive: 'main',
          resourceId: 'a'.repeat(32),
          gfsUri: `gfs://main/${'a'.repeat(32)}`,
          name: 'after-stop.bin',
          version: 1,
        },
        sizeBytes: 1,
        expiresAt: new Date(Date.now() + 60_000).toISOString(),
      })
    ).rejects.toMatchObject({ code: 'download_busy' })
  })

  it('the default cycle runs cleanup every five minutes', async () => {
    vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
    const cleanup = vi.spyOn(GfsDownloadStore.prototype, 'cleanupExpired')
    const runtime = await start(await root())
    expect(runtime.store.isAvailable()).toBe(true)

    await vi.advanceTimersByTimeAsync(5 * 60_000 - 1)
    expect(cleanup).not.toHaveBeenCalled()
    // Witness: the cycle reaches cleanup at five minutes, and again five minutes later.
    await vi.advanceTimersByTimeAsync(1)
    await vi.waitFor(() => expect(cleanup).toHaveBeenCalledTimes(1))
    // The next cycle is scheduled only after this one settles.
    await cleanup.mock.results[0]!.value
    await vi.advanceTimersByTimeAsync(5 * 60_000)
    await vi.waitFor(() => expect(cleanup).toHaveBeenCalledTimes(2))
  })

  it('stop resolves when the cycle in flight never settles: it waits stopTimeoutMs, warns and closes the store', async () => {
    const runtime = await bootstrapGfsRuntime(await root(), { cycleMs: 25, stopTimeoutMs: 50 })
    runtimes.push(runtime)
    const sweepStarted = deferred()
    vi.spyOn(GfsDownloadStore.prototype, 'cleanupExpired').mockImplementationOnce(() => {
      sweepStarted.resolve()
      return new Promise(() => undefined)
    })
    await sweepStarted.promise
    const close = vi.spyOn(runtime.store, 'close')
    const warn = vi.spyOn(logger, 'warn')

    await runtime.stop()

    expect(warn).toHaveBeenCalledWith(
      { component: 'gfs-runtime', timeoutMs: 50 },
      'GFS runtime stop timed out waiting for the cycle in flight; the store is closed anyway'
    )
    expect(close).toHaveBeenCalledWith(50)
    expect(runtime.store.isAvailable()).toBe(false)
  })

  it('stop during a slow cycle initialize leaves the store unavailable', async () => {
    const { link } = await symlinkedHostRoot()
    const realInitialize = GfsDownloadStore.prototype.initialize
    const retryStarted = deferred()
    const releaseRetry = deferred()
    let calls = 0
    vi.spyOn(GfsDownloadStore.prototype, 'initialize').mockImplementation(async function (
      this: GfsDownloadStore
    ) {
      calls += 1
      if (calls === 2) {
        retryStarted.resolve()
        await releaseRetry.promise
      }
      return realInitialize.call(this)
    })
    const runtime = await start(link, 25)
    expect(runtime.store.isAvailable()).toBe(false)
    await fs.rm(link)
    await fs.mkdir(link, { mode: 0o700 })
    await retryStarted.promise

    const stopping = runtime.stop()
    releaseRetry.resolve()
    await stopping

    // Witness: the slow retry ran to completion before the store was closed.
    expect(calls).toBe(2)
    expect(runtime.store.isAvailable()).toBe(false)
    await expect(runtime.store.initialize()).rejects.toMatchObject({ code: 'download_busy' })
  })
})