Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Restart behaviour of the filesystem GFS download store: a volume written by
* the ledger store of the pre-#1028 image (snapshots generated by running dev
* 74e0d81d9's store, see devGfsStoreFixture)
* never blocks admission, and a volume written by this store is swept but its
* copies are never reused or served after a restart.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import { type DevSnapshot, loadDevStore } from '../__tests__/fixtures/devGfsStoreFixture'
import {
callerDirectory,
completedCopy,
downloadDirectory,
exists,
expiryCount,
sourceFor,
} from '../__tests__/fixtures/gfsStoreTestKit'
import { logger } from '../logger'
import { isProtectedWorkspacePath } from '../workspace/protectedPaths'
import { GfsDownloadStore } from './gfsDownloadStore'
const { renameFailure, rmFailure } = vi.hoisted(() => ({
/** `target` is a Host-root entry name; the store renames it by its real path. */
renameFailure: { target: undefined as string | undefined, injected: 0 },
/** `prefix` is a Host-root entry name prefix the next `rm` refuses once. */
rmFailure: { prefix: undefined as string | undefined, injected: 0 },
}))
// Pass-through `rename` and `rm`: R3 makes the retirement rename of the legacy
// store fail once and R5 the removal of the retired tree, as a busy mount
// would (EBUSY: EACCES and EPERM from rm restore owner access and retry once).
vi.mock('node:fs/promises', async original => {
const actual = await original<typeof import('node:fs/promises')>()
const { freeSpaceSizedStatfs } = await import('../__tests__/fixtures/gfsStoreTestKit')
return {
...actual,
// The volume sized to its free space, so the disk's occupancy never
// meets the store's free-space floor.
statfs: freeSpaceSizedStatfs(actual.statfs),
rename: async (...values: Parameters<typeof actual.rename>) => {
if (
renameFailure.target !== undefined &&
String(values[0]).endsWith(`${path.sep}${renameFailure.target}`)
) {
renameFailure.target = undefined
renameFailure.injected += 1
throw Object.assign(new Error('simulated rename failure'), { code: 'EACCES' })
}
return actual.rename(...values)
},
rm: async (...values: Parameters<typeof actual.rm>) => {
if (
rmFailure.prefix !== undefined &&
path.basename(String(values[0])).startsWith(rmFailure.prefix)
) {
rmFailure.prefix = undefined
rmFailure.injected += 1
throw Object.assign(new Error('simulated rm failure'), { code: 'EBUSY' })
}
return actual.rm(...values)
},
}
})
const CALLER = 'caller-a'
const RETIRED_PREFIX = '.gfs-download-store.retired-'
const RETIREMENT_MESSAGE = 'Retired the pre-#1028 GFS download store'
const INPUT_RE = /^input-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
let hostRoot: string
const stores: GfsDownloadStore[] = []
async function open(): Promise<GfsDownloadStore> {
const store = new GfsDownloadStore(hostRoot)
stores.push(store)
await store.initialize()
return store
}
type LogCall = [Record<string, unknown>, string]
function startupLines(info: { mock: { calls: unknown[][] } }): Record<string, unknown>[] {
return (info.mock.calls as LogCall[])
.filter(([, message]) => message === 'GFS download store initialized')
.map(([fields]) => fields)
}
function lastStartupLine(info: { mock: { calls: unknown[][] } }): Record<string, unknown> {
const lines = startupLines(info)
if (lines.length === 0) throw new Error('the store logged no startup line')
return lines[lines.length - 1]!
}
function retirementWarnings(warn: { mock: { calls: unknown[][] } }): Record<string, unknown>[] {
return (warn.mock.calls as LogCall[])
.filter(([, message]) => message === RETIREMENT_MESSAGE)
.map(([fields]) => fields)
}
async function hostRootEntries(): Promise<string[]> {
return (await fs.readdir(hostRoot)).sort()
}
beforeEach(async () => {
hostRoot = await fs.realpath(await fs.mkdtemp(path.join(tmpdir(), 'gfs-store-restart-')))
})
afterEach(async () => {
renameFailure.target = undefined
renameFailure.injected = 0
rmFailure.prefix = undefined
rmFailure.injected = 0
for (const store of stores.splice(0)) await store.close()
vi.restoreAllMocks()
await fs.rm(hostRoot, { recursive: true, force: true })
})
describe('GFS download store restart', () => {
it.each([
['killed with an expired and a live processing lease', 'crashed', undefined],
['restarted by the dev Host, every record quarantined (#1019)', 'restarted', false],
] as Array<[string, DevSnapshot, boolean | undefined]>)(
'R1: restarting on a volume written by the dev store never blocks admission: %s',
async (_label, snapshot, oldStoreAvailable) => {
const dev = loadDevStore(hostRoot, snapshot)
// Witness: the old store, both of its leases and its copies are on disk
// before the start; on `restarted` the dev store itself was unavailable.
expect(exists(dev.storeRoot)).toBe(true)
expect(Date.parse(dev.leases.live.expiresAt)).toBeGreaterThan(Date.now())
expect(Date.parse(dev.leases.expired.expiresAt)).toBeLessThan(Date.now())
expect(dev.userDirectories.length).toBeGreaterThan(0)
expect(dev.oldStoreAvailable).toBe(oldStoreAvailable)
const warn = vi.spyOn(logger, 'warn')
const info = vi.spyOn(logger, 'info')
const retired = await expiryCount('retired_legacy_store')
const store = await open()
expect(store.isAvailable()).toBe(true)
const retirements = retirementWarnings(warn)
expect(retirements).toHaveLength(1)
expect(retirements[0]).toMatchObject({ component: 'GfsDownloadStore' })
expect(retirements[0]!.files as number).toBeGreaterThan(0)
expect(retirements[0]!.bytes as number).toBeGreaterThan(0)
expect(lastStartupLine(info)).toMatchObject({
retiredLegacyStore: true,
removedIncomplete: dev.userDirectories.length,
})
expect(await expiryCount('retired_legacy_store')).toBe(retired + 1)
expect(exists(dev.storeRoot)).toBe(false)
expect((await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))).toEqual([])
const { receipt, bytes } = await completedCopy(
store,
dev.callerRoot,
dev.callerIdentity,
1,
64
)
await expect(store.readManagedFile(receipt.path, dev.callerIdentity)).resolves.toEqual(bytes)
}
)
it('R2: dev user directories of every state are removed at startup and do not count against quota', async () => {
const dev = loadDevStore(hostRoot, 'crashed')
// Witness: every dev directory and its bytes are on disk before the start,
// with a completed, a missing and a transferring record among them.
expect(new Set(Object.values(dev.recordStates))).toEqual(
new Set(['completed', 'missing', 'transferring'])
)
expect(dev.userDirectories).toHaveLength(4)
expect(dev.userBytes).toBeGreaterThan(0)
for (const directory of dev.userDirectories) expect(exists(directory)).toBe(true)
const info = vi.spyOn(logger, 'info')
const store = await open()
expect(lastStartupLine(info)).toMatchObject({
removedIncomplete: 4,
retainedCompleted: 0,
retainedBytes: 0,
adopted: 0,
})
for (const directory of dev.userDirectories) expect(exists(directory)).toBe(false)
expect(await store.debugInventory()).toMatchObject({ bytes: 0, files: 0 })
const { bytes } = await completedCopy(store, dev.callerRoot, dev.callerIdentity, 2, 64)
expect(await store.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
})
it('R3: a failed retirement is logged, the store is available and the next sweep retires it', async () => {
const dev = loadDevStore(hostRoot, 'restarted')
const warn = vi.spyOn(logger, 'warn')
const removeFailed = await expiryCount('remove_failed')
renameFailure.target = '.gfs-download-store'
const store = await open()
// Witness: the retirement rename was attempted and failed exactly once.
expect(renameFailure.injected).toBe(1)
expect(warn).toHaveBeenCalledWith(
{ component: 'GfsDownloadStore', code: 'EACCES', step: 'rename' },
expect.stringContaining('could not retire')
)
expect(await expiryCount('remove_failed')).toBe(removeFailed + 1)
expect(store.isAvailable()).toBe(true)
expect(retirementWarnings(warn)).toEqual([])
expect(exists(dev.storeRoot)).toBe(true)
const root = callerDirectory(hostRoot, CALLER)
const { receipt, bytes } = await completedCopy(store, root, CALLER, 3, 64)
await expect(store.readManagedFile(receipt.path, CALLER)).resolves.toEqual(bytes)
// The periodic sweep retries without a restart.
await expect(store.cleanupExpired()).resolves.toMatchObject({ removeFailed: 0 })
expect(retirementWarnings(warn)).toHaveLength(1)
expect(exists(dev.storeRoot)).toBe(false)
expect(await hostRootEntries()).toEqual(['users'])
})
it('R4: after startup the volume has no store directory and only valid input directories', async () => {
const dev = loadDevStore(hostRoot, 'restarted')
const store = await open()
const published = await completedCopy(store, dev.callerRoot, dev.callerIdentity, 7, 64)
await completedCopy(store, callerDirectory(hostRoot, CALLER), CALLER, 8, 64)
expect(await hostRootEntries()).toEqual(['users'])
let directories = 0
for (const user of await fs.readdir(path.join(hostRoot, 'users'))) {
const downloads = path.join(hostRoot, 'users', user, '.gfs-downloads')
if (!exists(downloads)) continue
for (const name of await fs.readdir(downloads)) {
directories += 1
expect(name).toMatch(INPUT_RE)
const directory = path.join(downloads, name)
expect((await fs.readdir(directory)).sort()).toEqual(['meta.json', 'source'])
const meta = JSON.parse(await fs.readFile(path.join(directory, 'meta.json'), 'utf8'))
expect(meta).toMatchObject({ schemaVersion: 1, id: name.slice('input-'.length) })
const source = await fs.lstat(path.join(directory, 'source'))
expect(source.isFile()).toBe(true)
expect(source.size).toBe(meta.sizeBytes)
}
}
// Witness: the predicate walked both published downloads.
expect(directories).toBe(2)
expect(exists(downloadDirectory(dev.callerRoot, published.receipt.id))).toBe(true)
})
it('R5: a retired directory left by a failed removal is removed by the next sweep and is protected meanwhile', async () => {
const dev = loadDevStore(hostRoot, 'restarted')
const warn = vi.spyOn(logger, 'warn')
const removeFailed = await expiryCount('remove_failed')
rmFailure.prefix = RETIRED_PREFIX
const store = await open()
// Witness: the old store was renamed and the removal of the renamed tree failed once.
expect(rmFailure.injected).toBe(1)
expect(retirementWarnings(warn)).toHaveLength(1)
expect(warn).toHaveBeenCalledWith(
{ component: 'GfsDownloadStore', code: 'EBUSY', step: 'remove' },
expect.stringContaining('could not retire')
)
expect(await expiryCount('remove_failed')).toBe(removeFailed + 1)
expect(store.isAvailable()).toBe(true)
expect(exists(dev.storeRoot)).toBe(false)
const leftovers = (await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))
expect(leftovers).toHaveLength(1)
expect(isProtectedWorkspacePath(`${leftovers[0]}/ledger-v1.json`)).toBe(true)
await expect(store.cleanupExpired()).resolves.toMatchObject({ removeFailed: 0 })
expect(await hostRootEntries()).toEqual(['users'])
})
it('R6: a retired directory left by an earlier start is removed at startup and is protected meanwhile', async () => {
const leftover = `${RETIRED_PREFIX}00000000-0000-4000-8000-000000000000`
await fs.mkdir(path.join(hostRoot, leftover), { mode: 0o700 })
await fs.writeFile(path.join(hostRoot, leftover, 'ledger-v1.json'), '{}', { mode: 0o600 })
expect(isProtectedWorkspacePath(`${leftover}/ledger-v1.json`)).toBe(true)
expect(isProtectedWorkspacePath(leftover)).toBe(true)
// Witness: the check is a prefix match, not a blanket refusal.
expect(isProtectedWorkspacePath('.gfs-download-storex/ledger-v1.json')).toBe(false)
const info = vi.spyOn(logger, 'info')
const removeFailed = await expiryCount('remove_failed')
const store = await open()
expect(store.isAvailable()).toBe(true)
// Nothing named .gfs-download-store was found, so nothing was retired this start.
expect(lastStartupLine(info)).toMatchObject({ retiredLegacyStore: false })
expect(await expiryCount('remove_failed')).toBe(removeFailed)
expect(exists(path.join(hostRoot, leftover))).toBe(false)
expect(await hostRootEntries()).toEqual([])
})
it('R7: restarting with copies left by this design keeps them for accounting and does not reuse them', async () => {
const root = callerDirectory(hostRoot, CALLER)
const first = await open()
const { receipt, bytes } = await completedCopy(first, root, CALLER, 9, 64)
// Witness: before the restart the copy is reused and served.
await expect(first.reusableReceipt(CALLER, sourceFor(9), bytes.byteLength)).resolves.toEqual(
receipt
)
await first.close()
const info = vi.spyOn(logger, 'info')
const second = await open()
expect(lastStartupLine(info)).toMatchObject({
retiredLegacyStore: false,
removedIncomplete: 0,
retainedCompleted: 1,
retainedBytes: bytes.byteLength,
adopted: 1,
})
expect(await second.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
await expect(
second.reusableReceipt(CALLER, sourceFor(9), bytes.byteLength)
).resolves.toBeUndefined()
await expect(second.readManagedFile(receipt.path, CALLER)).rejects.toMatchObject({
code: 'download_missing',
})
expect(exists(downloadDirectory(root, receipt.id))).toBe(true)
})
it('R8: after a restart the same file downloads again under a new id that is reused and served', async () => {
const root = callerDirectory(hostRoot, CALLER)
const first = await open()
const before = await completedCopy(first, root, CALLER, 10, 64)
await first.close()
const second = await open()
const after = await completedCopy(second, root, CALLER, 10, 64)
expect(after.receipt.id).not.toBe(before.receipt.id)
await expect(second.readManagedFile(after.receipt.path, CALLER)).resolves.toEqual(after.bytes)
await expect(second.reusableReceipt(CALLER, sourceFor(10), 64)).resolves.toEqual(after.receipt)
// The adopted copy still counts until it expires or is evicted.
expect(await second.debugInventory()).toMatchObject({ bytes: 128, files: 2 })
})
})
describe('GFS download store restart: retirement edge cases', () => {
it('S14: only `.gfs-download-store.retired-<uuid>` names are removed', async () => {
const foreign = path.join(hostRoot, `${RETIRED_PREFIX}foo`)
const retired = path.join(hostRoot, `${RETIRED_PREFIX}00000000-0000-4000-8000-000000000001`)
for (const directory of [foreign, retired]) {
await fs.mkdir(directory, { mode: 0o700 })
await fs.writeFile(path.join(directory, 'copy'), 'bytes', { mode: 0o600 })
}
await open()
// Witness: a uuid-named retired tree is removed by the same sweep.
expect(exists(retired)).toBe(false)
expect(exists(path.join(foreign, 'copy'))).toBe(true)
})
it.each([
['a file', 'file'],
['a symlink', 'symlink'],
] as const)(
'S15: a `.gfs-download-store` that is %s is retired without following it',
async (_label, kind) => {
const legacy = path.join(hostRoot, '.gfs-download-store')
const outside = await fs.mkdtemp(path.join(tmpdir(), 'gfs-store-legacy-target-'))
const target = path.join(outside, 'target')
try {
await fs.writeFile(target, 'outside bytes', { mode: 0o600 })
if (kind === 'file') await fs.writeFile(legacy, 'legacy bytes', { mode: 0o600 })
else await fs.symlink(target, legacy)
const warn = vi.spyOn(logger, 'warn')
const retiredBefore = await expiryCount('retired_legacy_store')
const store = await open()
expect(store.isAvailable()).toBe(true)
expect(retirementWarnings(warn)).toEqual([
expect.objectContaining({ files: kind === 'file' ? 1 : 0 }),
])
expect(await expiryCount('retired_legacy_store')).toBe(retiredBefore + 1)
expect(exists(legacy)).toBe(false)
expect((await hostRootEntries()).filter(name => name.startsWith(RETIRED_PREFIX))).toEqual(
[]
)
// The symlink's target is never followed.
await expect(fs.readFile(target, 'utf8')).resolves.toBe('outside bytes')
} finally {
await fs.rm(outside, { recursive: true, force: true })
}
}
)
})