Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* The free-space floor (PR #1028 review round 2, A1-A3).
*
* The store keeps max(15% of the volume, 16 MiB) free. Admission refuses as
* disk_full when evicting unpinned copies cannot cover its request plus the
* floor, and startup and the periodic cleanupExpired evict unpinned copies
* when something else on the volume took the free space below the floor,
* until the floor plus 5% of the volume is free. statfs is the only
* boundary replaced: it reports a simulated 200 MiB volume (floor 30 MiB)
* whose free space is its size minus `foreign` bytes the store does not own
* minus the bytes of every regular file under the Host root, so a removal
* frees space exactly as on a disk.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import {
callerDirectory,
completedCopy,
downloadDirectory,
exists,
expiryCount,
metaFor,
plantDownload,
quotaCount,
sourceFor,
startTransfer,
} from '../__tests__/fixtures/gfsStoreTestKit'
import { logger } from '../logger'
import { GfsDownloadStore, freeSpaceFloor, freeSpaceRestoreTarget } from './gfsDownloadStore'
const { statfsBoundary } = vi.hoisted(() => ({ statfsBoundary: vi.fn() }))
vi.mock('node:fs/promises', async original => ({
...(await original<typeof fs>()),
statfs: statfsBoundary,
}))
const A = 'caller-a'
const B = 'caller-b'
const C = 'caller-c'
const MIB = 1024 * 1024
const HOUR = 60 * 60_000
const TOTAL = 200 * MIB
/** 15% of the 200 MiB volume: 31 457 280 bytes. */
const FLOOR = 30 * MIB
const FLOOR_WARNING =
'GFS download store found the volume below its free-space floor and evicted unpinned copies'
let nativeFs: typeof fs
let hostRoot: string
let rootA: string
let rootB: string
let rootC: string
let foreign = 0
const stores: GfsDownloadStore[] = []
function regularBytes(directory: string): number {
let sum = 0
for (const child of syncFs.readdirSync(directory, { withFileTypes: true })) {
const target = path.join(directory, child.name)
if (child.isDirectory()) sum += regularBytes(target)
else if (child.isFile()) sum += syncFs.lstatSync(target).size
}
return sum
}
function simulatedVolume(): void {
statfsBoundary.mockImplementation(async (target: string) => {
const real = await nativeFs.statfs(target, { bigint: true })
const free = BigInt(TOTAL - foreign - regularBytes(hostRoot))
return { ...real, bsize: 1n, blocks: BigInt(TOTAL), bavail: free < 0n ? 0n : free }
})
}
/** Sets the foreign data so the volume has exactly `freeBytes` free now. */
function leaveFree(freeBytes: number): void {
foreign = TOTAL - freeBytes - regularBytes(hostRoot)
}
async function openStore(): Promise<GfsDownloadStore> {
const opened = new GfsDownloadStore(hostRoot)
stores.push(opened)
await opened.initialize()
return opened
}
/** Distinct createdAt and last-use values so the eviction order is deterministic. */
function tick(): void {
vi.setSystemTime(Date.now() + 1_000)
}
function provenanceOf(store: GfsDownloadStore, id: string): string | undefined {
return (store as unknown as { entries: Map<string, { provenance: string }> }).entries.get(id)
?.provenance
}
function floorWarnings(warn: { mock: { calls: unknown[][] } }): unknown[] {
return warn.mock.calls.filter(call => call[1] === FLOOR_WARNING).map(call => call[0])
}
/** Splits `named` directories into the ones removed and the ones still on disk. */
function survey(named: Record<string, string>): { removed: string[]; kept: string[] } {
const removed: string[] = []
const kept: string[] = []
for (const [name, directory] of Object.entries(named))
(exists(directory) ? kept : removed).push(name)
return { removed: removed.sort(), kept: kept.sort() }
}
beforeEach(async () => {
nativeFs = await vi.importActual<typeof fs>('node:fs/promises')
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(new Date('2026-10-09T10:00:00.000Z'))
statfsBoundary.mockReset()
foreign = 0
simulatedVolume()
hostRoot = syncFs.mkdtempSync(path.join(tmpdir(), 'gfs-store-free-space-floor-'))
rootA = callerDirectory(hostRoot, A)
rootB = callerDirectory(hostRoot, B)
rootC = callerDirectory(hostRoot, C)
})
afterEach(async () => {
vi.restoreAllMocks()
vi.useRealTimers()
for (const opened of stores.splice(0)) await opened.close(0)
await nativeFs.rm(hostRoot, { recursive: true, force: true })
})
describe('GFS download store: free-space floor value', () => {
it('FSF-1: the floor is 15% of the volume, and never less than 16 MiB', () => {
// 10 GiB in 4 KiB blocks.
expect(freeSpaceFloor({ bsize: 4096n, blocks: 2_621_440n })).toBe(1_610_612_736n)
expect(freeSpaceFloor({ bsize: 1n, blocks: BigInt(TOTAL) })).toBe(BigInt(FLOOR))
// 15% of 100 MiB is 15 MiB, under the minimum.
expect(freeSpaceFloor({ bsize: 4096n, blocks: 25_600n })).toBe(16n * 1024n * 1024n)
expect(freeSpaceFloor({ bsize: 1n, blocks: 1000n })).toBe(16n * 1024n * 1024n)
expect(freeSpaceFloor({ bsize: 4096n, blocks: 0n })).toBe(16n * 1024n * 1024n)
})
it('FSF-1b: a restore aims for the floor plus 5% of the volume', () => {
expect(freeSpaceRestoreTarget({ bsize: 1n, blocks: BigInt(TOTAL) })).toBe(
BigInt(FLOOR + 10 * MIB)
)
// 16 MiB minimum floor plus 5% of 1000 bytes.
expect(freeSpaceRestoreTarget({ bsize: 1n, blocks: 1000n })).toBe(16n * 1024n * 1024n + 50n)
})
})
describe('GFS download store: admission keeps the free-space floor', () => {
it('FSF-2: an admission is refused as disk_full while every copy is pinned, and admitted after evicting released copies', async () => {
const store = await openStore()
const ids: string[] = []
for (let index = 0; index < 10; index += 1) {
const { receipt } = await completedCopy(store, rootB, B, 100 + index, MIB, {
owner: 'task-b',
})
ids.push(receipt.id)
tick()
}
// 30 MiB free: a 5 MiB request needs 5 MiB plus the 30 MiB floor.
leaveFree(FLOOR)
const diskDenied = await quotaCount('host', 'free_space')
const evicted = await expiryCount('expired_removed')
await expect(startTransfer(store, rootA, A, 200, 5 * MIB)).rejects.toMatchObject({
code: 'disk_full',
})
expect(await quotaCount('host', 'free_space')).toBe(diskDenied + 1)
expect(ids.every(id => exists(downloadDirectory(rootB, id)))).toBe(true)
expect(await expiryCount('expired_removed')).toBe(evicted)
await store.releaseReceiptOwner('task-b', B)
const admitted = await startTransfer(store, rootA, A, 201, 5 * MIB)
// Witness: the admitted transfer exists and the five least recently used
// copies were evicted to cover its 5 MiB; the other five stay.
expect(exists(downloadDirectory(rootA, admitted.transfer.id))).toBe(true)
expect(ids.map(id => exists(downloadDirectory(rootB, id)))).toEqual([
false,
false,
false,
false,
false,
true,
true,
true,
true,
true,
])
expect(await expiryCount('expired_removed')).toBe(evicted + 5)
await store.fail(admitted.transfer.id, A)
})
})
describe('GFS download store: the periodic sweep restores the free-space floor', () => {
it('FSF-3: below the floor, cleanupExpired evicts the adopted copy and then the least recently used ones up to the floor plus 5%, and keeps pinned copies and active transfers', async () => {
const plantedBytes = Buffer.alloc(4 * MIB, 3)
const plantedId = randomUUID()
const planted = plantDownload(rootC, {
id: plantedId,
bytes: plantedBytes,
meta: metaFor(plantedId, plantedBytes, Date.now(), {
callerIdentity: C,
source: sourceFor(300),
expiresAt: new Date(Date.now() + 2 * HOUR).toISOString(),
}),
})
const store = await openStore()
expect(provenanceOf(store, planted.id)).toBe('adopted')
tick()
// The pinned copy is the oldest published one, so only its pin keeps it.
const pinned = await completedCopy(store, rootA, A, 301, 4 * MIB, { owner: 'task-a' })
tick()
const oldest = await completedCopy(store, rootB, B, 302, 4 * MIB)
tick()
const middle = await completedCopy(store, rootB, B, 303, 4 * MIB)
tick()
const newest = await completedCopy(store, rootB, B, 304, 4 * MIB)
tick()
const active = await startTransfer(store, rootA, A, 305, 4 * MIB)
// 1 MiB below the 30 MiB floor; the restore aims for 30 + 10 MiB free, so
// it needs 11 MiB: the adopted copy, then the oldest and the middle one.
leaveFree(FLOOR - MIB)
const warn = vi.spyOn(logger, 'warn')
const evicted = await expiryCount('expired_removed')
const result = await store.cleanupExpired()
expect(result).toEqual({ removedExpired: 0, removedIncomplete: 0, removeFailed: 0 })
expect(
survey({
adopted: planted.directory,
pinned: downloadDirectory(rootA, pinned.receipt.id),
oldest: downloadDirectory(rootB, oldest.receipt.id),
middle: downloadDirectory(rootB, middle.receipt.id),
newest: downloadDirectory(rootB, newest.receipt.id),
active: downloadDirectory(rootA, active.transfer.id),
})
).toEqual({ removed: ['adopted', 'middle', 'oldest'], kept: ['active', 'newest', 'pinned'] })
expect(floorWarnings(warn)).toEqual([
expect.objectContaining({ evicted: 3, removeFailed: 0, restored: true }),
])
expect(await expiryCount('expired_removed')).toBe(evicted + 3)
// The kept copies are still served.
expect((await store.readManagedFile(pinned.receipt.path, A)).equals(pinned.bytes)).toBe(true)
await store.fail(active.transfer.id, A)
})
it('FSF-4: above the floor, cleanupExpired evicts nothing while it still removes an expired copy', async () => {
const store = await openStore()
const expiring = await completedCopy(store, rootB, B, 400, MIB, {
expiresAt: new Date(Date.now() + HOUR).toISOString(),
})
tick()
const live = await completedCopy(store, rootB, B, 401, MIB)
leaveFree(FLOOR + 10 * MIB)
vi.setSystemTime(Date.now() + 90 * 60_000)
const warn = vi.spyOn(logger, 'warn')
const evicted = await expiryCount('expired_removed')
const result = await store.cleanupExpired()
// Witness: the same call removed the expired copy.
expect(result).toEqual({ removedExpired: 1, removedIncomplete: 0, removeFailed: 0 })
expect(exists(downloadDirectory(rootB, expiring.receipt.id))).toBe(false)
expect(await expiryCount('expired_removed')).toBe(evicted + 1)
expect(exists(downloadDirectory(rootB, live.receipt.id))).toBe(true)
expect(floorWarnings(warn)).toEqual([])
})
it('FSF-5: a deficit larger than every unpinned copy is restored partially, and the evictions are kept', async () => {
const store = await openStore()
const pinned = await completedCopy(store, rootA, A, 500, MIB, { owner: 'task-a' })
tick()
const first = await completedCopy(store, rootB, B, 501, MIB)
tick()
const second = await completedCopy(store, rootB, B, 502, MIB)
leaveFree(FLOOR - 5 * MIB)
const warn = vi.spyOn(logger, 'warn')
const evicted = await expiryCount('expired_removed')
await expect(store.cleanupExpired()).resolves.toEqual({
removedExpired: 0,
removedIncomplete: 0,
removeFailed: 0,
})
expect(
survey({
pinned: downloadDirectory(rootA, pinned.receipt.id),
first: downloadDirectory(rootB, first.receipt.id),
second: downloadDirectory(rootB, second.receipt.id),
})
).toEqual({ removed: ['first', 'second'], kept: ['pinned'] })
expect(floorWarnings(warn)).toEqual([
expect.objectContaining({ evicted: 2, removeFailed: 0, restored: false }),
])
expect(await expiryCount('expired_removed')).toBe(evicted + 2)
})
it.each([
['statfs rejects', 'EIO'],
['two invalid readings', 'volume_unmeasurable'],
])(
'FSF-6: when %s, cleanupExpired keeps its sweep result, evicts nothing and logs the failure',
async (_case, code) => {
const store = await openStore()
const expiring = await completedCopy(store, rootB, B, 600, MIB, {
expiresAt: new Date(Date.now() + HOUR).toISOString(),
})
tick()
const unpinned = await completedCopy(store, rootB, B, 601, MIB)
leaveFree(FLOOR - 5 * MIB)
vi.setSystemTime(Date.now() + 90 * 60_000)
statfsBoundary.mockReset()
if (code === 'EIO')
statfsBoundary.mockRejectedValue(Object.assign(new Error('simulated'), { code: 'EIO' }))
else
statfsBoundary.mockImplementation(async (target: string) => ({
...(await nativeFs.statfs(target, { bigint: true })),
blocks: 0n,
}))
const warn = vi.spyOn(logger, 'warn')
const failed = await expiryCount('sweep_failed')
const result = await store.cleanupExpired()
expect(result).toEqual({ removedExpired: 1, removedIncomplete: 0, removeFailed: 0 })
expect(exists(downloadDirectory(rootB, expiring.receipt.id))).toBe(false)
// Witness: the restore ran and could not measure the volume.
expect(statfsBoundary).toHaveBeenCalled()
expect(await expiryCount('sweep_failed')).toBe(failed + 1)
expect(warn).toHaveBeenCalledWith(
expect.objectContaining({ code }),
'GFS download store could not measure the volume to restore its free-space floor; the next sweep retries it'
)
expect(exists(downloadDirectory(rootB, unpinned.receipt.id))).toBe(true)
expect(floorWarnings(warn)).toEqual([])
}
)
it('FSF-7: a restore continues past the floor and stops once the floor plus 5% is free', async () => {
const store = await openStore()
const ids: string[] = []
for (let index = 0; index < 5; index += 1) {
const { receipt } = await completedCopy(store, rootB, B, 700 + index, 3 * MIB)
ids.push(receipt.id)
tick()
}
// One byte below the floor. Restoring only the floor would take the
// oldest copy; the floor plus 10 MiB needs 10 MiB + 1 byte: four copies.
leaveFree(FLOOR - 1)
const warn = vi.spyOn(logger, 'warn')
await store.cleanupExpired()
expect(ids.map(id => exists(downloadDirectory(rootB, id)))).toEqual([
false,
false,
false,
false,
true,
])
expect(floorWarnings(warn)).toEqual([
expect.objectContaining({ evicted: 4, removeFailed: 0, restored: true }),
])
})
})
describe('GFS download store: startup restores the free-space floor', () => {
it('FSF-8: a store initialized on a volume below the floor evicts adopted copies before it is available', async () => {
const first = await openStore()
const ids: string[] = []
for (let index = 0; index < 4; index += 1) {
const { receipt } = await completedCopy(first, rootB, B, 800 + index, 4 * MIB)
ids.push(receipt.id)
tick()
}
stores.splice(stores.indexOf(first), 1)
await first.close(0)
// 1 MiB below the floor at restart: the restore needs 11 MiB, three copies.
leaveFree(FLOOR - MIB)
const warn = vi.spyOn(logger, 'warn')
const evicted = await expiryCount('expired_removed')
statfsBoundary.mockClear()
const restarted = await openStore()
expect(restarted.isAvailable()).toBe(true)
// Witness: initialize measured the volume and logged the restore.
expect(statfsBoundary).toHaveBeenCalled()
expect(floorWarnings(warn)).toEqual([
expect.objectContaining({ evicted: 3, removeFailed: 0, restored: true }),
])
expect(ids.filter(id => exists(downloadDirectory(rootB, id)))).toHaveLength(1)
expect(await expiryCount('expired_removed')).toBe(evicted + 3)
})
})