Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import * as syncFs from 'node:fs'
import * as fs from 'node:fs/promises'
import { tmpdir } from 'node:os'
import * as path from 'node:path'
import { setTimeout as delay } from 'node:timers/promises'
import { register } from 'prom-client'
import { downloadGfsContent } from './gfsContentDownload'
import { readGfsMetadata } from './gfsContentRead'
import { GfsDownloadStore } from './gfsDownloadStore'
const { rmFailure } = vi.hoisted(() => ({
/** `target` is a name prefix under `.gfs-downloads`, e.g. the store's `.trash-` names. */
rmFailure: { target: undefined as string | undefined, injected: 0 },
}))
// Pass-through `rm`: one test makes the store's removal of a transfer
// directory fail once, as a read-only parent or a busy mount would.
vi.mock('node:fs/promises', async original => {
const actual = await original<typeof import('node:fs/promises')>()
// statfs reports the volume sized to its free space, so the disk's occupancy
// never meets the store's free-space floor.
const { freeSpaceSizedStatfs } = await import('../__tests__/fixtures/gfsStoreTestKit')
return {
...actual,
statfs: freeSpaceSizedStatfs(actual.statfs),
rm: async (...values: Parameters<typeof actual.rm>) => {
if (
rmFailure.target !== undefined &&
String(values[0]).includes(`/.gfs-downloads/${rmFailure.target}`)
) {
rmFailure.target = undefined
rmFailure.injected += 1
// EBUSY, not EACCES: a permission error is retried after restoring owner access.
throw Object.assign(new Error('simulated rm failure'), { code: 'EBUSY' })
}
return actual.rm(...values)
},
}
})
const args = { drive: 'main', resourceId: 'a'.repeat(32) }
const uri = `gfs://main/${args.resourceId}`
let hostRoot: string
let callerRoot: string
let store: GfsDownloadStore
function metadata(size: number, overrides: Record<string, unknown> = {}): Response {
return new Response(
JSON.stringify({
ok: true,
data: {
...args,
rid: args.resourceId,
gfsUri: uri,
kind: 'file',
name: 'input.csv',
version: 4,
bytes: size,
...overrides,
},
}),
{ headers: { 'content-type': 'application/json' } }
)
}
function contentHeaders(size: number, overrides: Record<string, string | undefined> = {}) {
const defined = Object.fromEntries(
Object.entries(overrides).filter(([, value]) => value !== undefined)
) as Record<string, string>
const headers = {
'content-length': String(size),
'x-gfs-uri': uri,
'x-gfs-version': '4',
...defined,
}
if ('content-length' in overrides && overrides['content-length'] === undefined)
delete (headers as { 'content-length'?: string })['content-length']
return headers
}
function harness(
bytes: Uint8Array,
changes: {
metadata?: Record<string, unknown>
headers?: Record<string, string | undefined>
body?: ReadableStream<Uint8Array>
} = {}
) {
return vi.fn(async (url: string) =>
url.includes('/content?')
? new Response(changes.body ?? new Uint8Array(bytes), {
headers: contentHeaders(bytes.byteLength, changes.headers),
})
: metadata(bytes.byteLength, changes.metadata)
)
}
function options(overrides: Record<string, unknown> = {}) {
return {
store,
callerIdentity: 'caller-a',
callerWorkspacePath: callerRoot,
...overrides,
}
}
async function expiryCount(outcome: string): Promise<number> {
const metric = register.getSingleMetric('clerum_gfs_download_expiry_total')
if (metric === undefined)
throw new Error('metric clerum_gfs_download_expiry_total is not registered')
const { values } = await metric.get()
return values
.filter(sample => sample.labels.outcome === outcome)
.reduce((sum, sample) => sum + sample.value, 0)
}
async function retainedDirectories(): Promise<string[]> {
return fs.readdir(path.join(callerRoot, '.gfs-downloads')).catch(() => [])
}
beforeEach(async () => {
hostRoot = await fs.mkdtemp(path.join(tmpdir(), 'gfs-download-test-'))
callerRoot = path.join(hostRoot, 'users', 'caller-a')
await fs.mkdir(callerRoot, { recursive: true, mode: 0o700 })
store = new GfsDownloadStore(hostRoot)
await store.initialize()
})
afterEach(async () => {
rmFailure.target = undefined
rmFailure.injected = 0
vi.useRealTimers()
vi.restoreAllMocks()
await fs.rm(hostRoot, { recursive: true, force: true })
})
describe('governed GFS content download', () => {
it('streams the exact incident-sized CSV and returns only a bounded receipt', async () => {
const bytes = Buffer.alloc(3_836_961, 65)
const result = await downloadGfsContent(harness(bytes), args, options())
expect(await fs.readFile(path.join(callerRoot, result.path))).toEqual(bytes)
expect(result).toMatchObject({
sizeBytes: 3_836_961,
sha256: createHash('sha256').update(bytes).digest('hex'),
})
expect(result.source.version).toBe(4)
expect(JSON.stringify(result)).not.toContain('AAAAAAA')
expect(JSON.stringify(result).length).toBeLessThan(4096)
expect((await fs.stat(path.join(callerRoot, result.path))).mode & 0o777).toBe(0o600)
expect((await fs.stat(path.dirname(path.join(callerRoot, result.path)))).mode & 0o777).toBe(
0o700
)
// The durable 3.8 MB publication measured 7.7 s on a loaded host.
}, 30_000)
it('waits for each partial write before pulling more content', async () => {
let finish!: () => void
const finalChunk = new Promise<void>(resolve => {
finish = resolve
})
let firstWritten!: () => void
const started = new Promise<void>(resolve => {
firstWritten = resolve
})
let pulls = 0
const body = new ReadableStream<Uint8Array>(
{
async pull(controller) {
if (pulls++ === 0) {
controller.enqueue(new Uint8Array([1, 2, 3]))
return
}
await finalChunk
controller.enqueue(new Uint8Array([4, 5, 6]))
controller.close()
},
},
{ highWaterMark: 0 }
)
const probe = await fs.open(path.join(callerRoot, 'probe'), 'w')
const prototype = Object.getPrototypeOf(probe)
await probe.close()
const original = prototype.write
vi.spyOn(prototype, 'write').mockImplementation(async function (
this: fs.FileHandle,
...values: unknown[]
) {
const written = await original.apply(this, values)
firstWritten()
return written
})
const pending = downloadGfsContent(harness(new Uint8Array(6), { body }), args, options())
await started
const [directory] = await retainedDirectories()
expect(
await fs.readFile(path.join(callerRoot, '.gfs-downloads', directory, 'source.partial'))
).toEqual(Buffer.from([1, 2, 3]))
finish()
const result = await pending
expect(await fs.readFile(path.join(callerRoot, result.path))).toEqual(
Buffer.from([1, 2, 3, 4, 5, 6])
)
})
it('rejects a source over the configured general limit before content', async () => {
const request = harness(new Uint8Array(0), {
metadata: { bytes: 16 * 1024 * 1024 + 1 },
})
await expect(downloadGfsContent(request, args, options())).rejects.toMatchObject({
code: 'limit_exceeded',
})
expect(request).toHaveBeenCalledTimes(1)
})
it.each([
[{ 'x-gfs-version': '5' }, 'version_conflict'],
[{ 'x-gfs-uri': 'gfs://main/other' }, 'identity_mismatch'],
[{ 'content-length': '4' }, 'incomplete_response'],
[{ 'content-encoding': 'gzip' }, 'invalid_response'],
[{ 'content-length': undefined }, 'incomplete_response'],
])('does not publish inconsistent content headers: %j', async (changed, code) => {
await expect(
downloadGfsContent(
harness(new Uint8Array(3), {
headers: changed,
}),
args,
options()
)
).rejects.toMatchObject({ code })
expect(await retainedDirectories()).toEqual([])
})
it.each([
[2, 'incomplete_response'],
[4, 'limit_exceeded'],
])('removes truncated or oversized content (%i bytes)', async (length, code) => {
const request = vi.fn(async (url: string) =>
url.includes('/content?')
? new Response(new Uint8Array(length), { headers: contentHeaders(3) })
: metadata(3)
)
await expect(downloadGfsContent(request, args, options())).rejects.toMatchObject({ code })
expect(await retainedDirectories()).toEqual([])
})
it('cancels an in-progress stream, removes the partial, and frees the slot', async () => {
const controller = new AbortController()
let pulled!: () => void
const started = new Promise<void>(resolve => {
pulled = resolve
})
let count = 0
const body = new ReadableStream<Uint8Array>(
{
pull(stream) {
if (count++ === 0) stream.enqueue(new Uint8Array([1]))
else pulled()
},
},
{ highWaterMark: 0 }
)
const pending = downloadGfsContent(
harness(new Uint8Array(2), { body }),
args,
options({ signal: controller.signal })
)
const rejected = expect(pending).rejects.toMatchObject({ code: 'cancelled' })
await started
controller.abort()
await rejected
expect(await retainedDirectories()).toEqual([])
await downloadGfsContent(harness(new Uint8Array([1])), args, options())
})
it('enforces an absolute deadline when the request callback ignores abort', async () => {
vi.useFakeTimers()
const request = vi.fn(() => new Promise<Response>(() => undefined))
const rejected = expect(
downloadGfsContent(request, args, options({ timeoutMs: 10 }))
).rejects.toMatchObject({ code: 'timeout' })
await vi.advanceTimersByTimeAsync(11)
await rejected
})
it('applies its own metadata deadline even without an external signal', async () => {
vi.useFakeTimers()
const request = vi.fn(() => new Promise<Response>(() => undefined))
const rejected = expect(
readGfsMetadata(request, args, { timeoutMs: 10 })
).rejects.toMatchObject({ code: 'timeout' })
await vi.advanceTimersByTimeAsync(11)
await rejected
expect(request).toHaveBeenCalledTimes(1)
})
it('rejects redirects and preserves authoritative GFSC denials', async () => {
const redirected = metadata(0)
Object.defineProperty(redirected, 'redirected', { value: true })
await expect(
downloadGfsContent(
vi.fn(async () => redirected),
args,
options()
)
).rejects.toMatchObject({ code: 'invalid_response' })
await expect(
downloadGfsContent(
vi.fn(async () => new Response('forbidden', { status: 403 })),
args,
options()
)
).rejects.toThrow('gfsc 403')
expect(await retainedDirectories()).toEqual([])
})
it('does not publish when file sync is released after abort', async () => {
const controller = new AbortController()
const bytes = new Uint8Array([1, 2, 3, 4, 5])
const probe = await fs.open(path.join(callerRoot, 'probe'), 'w')
const prototype = Object.getPrototypeOf(probe) as fs.FileHandle
await probe.close()
let releaseSync!: () => void
const syncBlocked = new Promise<void>(resolve => {
releaseSync = resolve
})
let syncEntered!: () => void
const syncStarted = new Promise<void>(resolve => {
syncEntered = resolve
})
let heldSync = false
const originalSync = prototype.sync
vi.spyOn(prototype, 'sync').mockImplementation(async function (this: fs.FileHandle) {
const info = await this.stat()
if (!heldSync && info.isFile() && info.size === bytes.byteLength) {
heldSync = true
syncEntered()
await syncBlocked
}
return originalSync.call(this)
})
const pending = downloadGfsContent(harness(bytes), args, options({ signal: controller.signal }))
const rejected = expect(pending).rejects.toMatchObject({ code: 'cancelled' })
await syncStarted
controller.abort()
releaseSync()
await rejected
expect(await retainedDirectories()).toEqual([])
expect(await store.debugInventory()).toMatchObject({ bytes: 0, files: 0 })
await downloadGfsContent(harness(new Uint8Array([1])), args, options())
})
it('does not commit a publication released after abort', async () => {
const controller = new AbortController()
const bytes = new Uint8Array([9, 8, 7])
const probe = await fs.open(path.join(callerRoot, 'probe'), 'w')
const prototype = Object.getPrototypeOf(probe) as fs.FileHandle
await probe.close()
let releaseRead!: () => void
const readBlocked = new Promise<void>(resolve => {
releaseRead = resolve
})
let readEntered!: () => void
const readStarted = new Promise<void>(resolve => {
readEntered = resolve
})
let heldRead = false
const originalRead = prototype.read
vi.spyOn(prototype, 'read').mockImplementation(async function (
this: fs.FileHandle,
...values: unknown[]
) {
const info = await this.stat()
if (!heldRead && info.isFile() && info.size === bytes.byteLength) {
heldRead = true
readEntered()
await readBlocked
}
return originalRead.apply(this, values as never)
})
const pending = downloadGfsContent(harness(bytes), args, options({ signal: controller.signal }))
const rejected = expect(pending).rejects.toMatchObject({ code: 'cancelled' })
await readStarted
controller.abort()
releaseRead()
await rejected
expect(await retainedDirectories()).toEqual([])
expect(await store.debugInventory()).toMatchObject({ bytes: 0, files: 0 })
await downloadGfsContent(harness(new Uint8Array([1])), args, options())
})
it('releases the reservation when cancellation cleanup fails and a later sweep removes the leftover', async () => {
const controller = new AbortController()
const bytes = new Uint8Array([1, 2, 3, 4, 5])
const probe = await fs.open(path.join(callerRoot, 'probe'), 'w')
const prototype = Object.getPrototypeOf(probe) as fs.FileHandle
await probe.close()
let releaseSync!: () => void
const syncBlocked = new Promise<void>(resolve => {
releaseSync = resolve
})
let syncEntered!: () => void
const syncStarted = new Promise<void>(resolve => {
syncEntered = resolve
})
let heldSync = false
const originalSync = prototype.sync
vi.spyOn(prototype, 'sync').mockImplementation(async function (this: fs.FileHandle) {
const info = await this.stat()
if (!heldSync && info.isFile() && info.size === bytes.byteLength) {
heldSync = true
syncEntered()
await syncBlocked
}
return originalSync.call(this)
})
const pending = downloadGfsContent(harness(bytes), args, options({ signal: controller.signal }))
const rejected = expect(pending).rejects.toMatchObject({ code: 'cancelled' })
await syncStarted
const [name] = await retainedDirectories()
const removeFailed = await expiryCount('remove_failed')
// The store renames a directory to a private `.trash-<uuid>` name before rm.
rmFailure.target = '.trash-'
controller.abort()
releaseSync()
await rejected
// Witness: the store's removal was attempted and failed exactly once.
expect(rmFailure.injected).toBe(1)
expect(await expiryCount('remove_failed')).toBe(removeFailed + 1)
// The leftover is the renamed transfer: incomplete and holding no reservation.
const retained = await retainedDirectories()
expect(retained).not.toContain(name)
expect(retained).toHaveLength(1)
expect(retained[0]).toMatch(/^\.trash-[0-9a-f-]{36}$/)
const directory = path.join(callerRoot, '.gfs-downloads', retained[0]!)
await expect(fs.lstat(directory)).resolves.toBeDefined()
await expect(fs.lstat(path.join(directory, 'meta.json'))).rejects.toMatchObject({
code: 'ENOENT',
})
expect(await store.debugInventory()).toMatchObject({ bytes: 0, files: 0 })
await expect(
downloadGfsContent(harness(new Uint8Array([1])), args, options())
).resolves.toMatchObject({
sizeBytes: 1,
})
// The next admission read the disk and removed the leftover.
await expect(fs.lstat(directory)).rejects.toMatchObject({ code: 'ENOENT' })
})
it('reuses a verified copy only after fresh authorization without renewing its TTL', async () => {
const bytes = new Uint8Array([1, 2, 3, 4])
const request = vi.fn(harness(bytes))
const first = await downloadGfsContent(request, args, options())
const usage = await store.debugInventory()
const second = await downloadGfsContent(request, args, options())
expect(second).toEqual(first)
expect(second.expiresAt).toBe(first.expiresAt)
expect(await store.debugInventory()).toEqual(usage)
expect(request.mock.calls).toHaveLength(3)
expect(request.mock.calls.filter(([url]) => url.includes('/content?'))).toHaveLength(1)
})
it('does not reuse retained bytes after upstream permission is revoked', async () => {
const bytes = new Uint8Array([1, 2, 3])
await downloadGfsContent(harness(bytes), args, options())
const request = vi.fn(async () => new Response('denied', { status: 403 }))
await expect(downloadGfsContent(request, args, options())).rejects.toThrow('gfsc 403')
expect(request).toHaveBeenCalledOnce()
expect(await store.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
})
it('does not substitute a retained version for a newly stale reference', async () => {
const bytes = new Uint8Array([1, 2, 3])
await downloadGfsContent(harness(bytes), args, options({ expectedVersion: 4 }))
const request = vi.fn(harness(bytes, { metadata: { version: 5 } }))
await expect(
downloadGfsContent(request, args, options({ expectedVersion: 4 }))
).rejects.toMatchObject({
code: 'version_conflict',
})
expect(request).toHaveBeenCalledOnce()
})
it('replaces an invalid cached copy through the authorized source and removes the invalid copy', async () => {
const bytes = new Uint8Array([1, 2, 3])
const first = await downloadGfsContent(harness(bytes), args, options())
await fs.writeFile(path.join(callerRoot, first.path), new Uint8Array([3, 2, 1]))
const request = vi.fn(harness(bytes))
const second = await downloadGfsContent(request, args, options())
expect(second.id).not.toBe(first.id)
expect(second.sha256).toBe(first.sha256)
expect(request).toHaveBeenCalledTimes(2)
await expect(
fs.lstat(path.join(callerRoot, '.gfs-downloads', `input-${first.id}`))
).rejects.toMatchObject({ code: 'ENOENT' })
expect(await store.debugInventory()).toMatchObject({ bytes: bytes.byteLength, files: 1 })
await expect(store.readManagedFile(second.path, 'caller-a')).resolves.toEqual(
Buffer.from(bytes)
)
})
it('fails at once when source.partial is replaced by a FIFO before the download opens it', async () => {
const createTransfer = store.createTransfer.bind(store)
let fifo: string | undefined
vi.spyOn(store, 'createTransfer').mockImplementationOnce(async input => {
const transfer = await createTransfer(input)
fifo = path.join(callerRoot, transfer.partialPath)
await fs.rm(fifo)
execFileSync('mkfifo', [fifo])
return transfer
})
const bytes = Buffer.from('fifo partial')
const download = downloadGfsContent(harness(bytes), args, options())
const observed = download.then(
() => 'downloaded',
() => 'refused'
)
const outcome = await Promise.race([observed, delay(2_000).then(() => 'blocked')])
if (outcome === 'blocked' && fifo !== undefined) {
// Release the blocked open so the run ends; the assertion below fails.
syncFs.closeSync(
syncFs.openSync(fifo, syncFs.constants.O_RDONLY | syncFs.constants.O_NONBLOCK)
)
await observed
}
// Witness: the partial really was a FIFO when the download opened it.
expect(fifo).toBeDefined()
expect(outcome).toBe('refused')
await expect(download).rejects.toMatchObject({ code: 'storage_write_failed' })
// The next download of the same file works.
const next = await downloadGfsContent(harness(bytes), args, options())
await expect(store.readManagedFile(next.path, 'caller-a')).resolves.toEqual(bytes)
})
})