Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { createHash, randomBytes, randomUUID } from 'node:crypto'
import type { Readable } from 'node:stream'
import { Pool, type PoolClient } from 'pg'
import { PgBlobStagingStore } from '../../gfs-controller/src/db/blobStaging.js'
import {
type BlobWriter,
GfsWriteService,
type Transactor,
} from '../../gfs-controller/src/db/writeStore.js'
import { initDb } from '../src/db.js'
import type { GfsCaller } from '../src/routes/gfs/grants.js'
import { applyResourcePatch } from '../src/routes/gfs/resources.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
const operator: GfsCaller = {
isOperator: true,
subjects: new Set(['operator:']),
actorKey: 'operator:',
}
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
url.pathname = `/${database}`
return url.toString()
}
function quoteIdent(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
async function begin(client: PoolClient): Promise<void> {
await client.query('BEGIN')
await client.query(`SET LOCAL lock_timeout = '5s'`)
await client.query(`SET LOCAL statement_timeout = '10s'`)
}
async function waitUntilBlocked(
pool: Pool,
blockedPid: number,
blockerPid: number,
label: string
): Promise<void> {
const deadline = Date.now() + 5_000
while (Date.now() < deadline) {
const result = await pool.query<{ blocked: boolean }>(
`SELECT $2::integer = ANY(pg_blocking_pids($1::integer)) AS blocked`,
[blockedPid, blockerPid]
)
if (result.rows[0]?.blocked) return
await new Promise<void>(resolve => setImmediate(resolve))
}
throw new Error(`${label}: expected PostgreSQL blocker was not observed`)
}
interface Tree {
drive: string
root: string
source: string
destinationA: string
destinationB: string
child: string
}
async function seedTree(pool: Pool, label: string): Promise<Tree> {
const tree: Tree = {
drive: `real-pg-${label}-${randomBytes(4).toString('hex')}`,
root: randomUUID(),
source: randomUUID(),
destinationA: randomUUID(),
destinationB: randomUUID(),
child: randomUUID(),
}
await pool.query(
`INSERT INTO gfs_resources
(resource_id, drive, parent_resource_id, name, kind, path_cache, version, bytes)
VALUES ($1,$6,NULL,'','directory','/',0,0),
($2,$6,$1,'source','directory','/source',0,0),
($3,$6,$1,'destination-a','directory','/destination-a',0,0),
($4,$6,$1,'destination-b','directory','/destination-b',0,0),
($5,$6,$2,'child.txt','file','/source/child.txt',0,0)`,
[tree.root, tree.source, tree.destinationA, tree.destinationB, tree.child, tree.drive]
)
return tree
}
async function move(client: PoolClient, tree: Tree, destination: string): Promise<number> {
const result = await applyResourcePatch(
client,
operator,
tree.drive,
tree.child,
{ newParentId: destination },
async () => undefined
)
return result.version
}
const blobs: BlobWriter = {
async writeImmutable(resourceId: string, generation: string, data: Readable | Buffer) {
if (!Buffer.isBuffer(data)) throw new Error('real-PG fixture expects buffered replacement')
return {
blobKey: `${resourceId.replaceAll('-', '')}/${generation}`,
bytes: data.length,
contentSha256: createHash('sha256').update(data).digest('hex'),
}
},
async verify() {},
async deleteByKey() {},
async deleteLegacyFlat() {},
}
function boundTransaction(client: PoolClient): Transactor {
return { transaction: work => work(client) }
}
function writer(client: PoolClient, manifests = new PgBlobStagingStore(client)): GfsWriteService {
return new GfsWriteService(boundTransaction(client), blobs, manifests)
}
function deferred(): { promise: Promise<void>; resolve: () => void } {
let resolve!: () => void
const promise = new Promise<void>(done => {
resolve = done
})
return { promise, resolve }
}
async function expectNoLiveNodeBelowTombstone(pool: Pool): Promise<void> {
const result = await pool.query<{ violations: string }>(`WITH RECURSIVE ancestry AS (
SELECT resource_id AS live_id,parent_resource_id,ARRAY[resource_id]::uuid[] visited
FROM gfs_resources WHERE deleted_at IS NULL UNION ALL
SELECT a.live_id,p.parent_resource_id,a.visited||p.resource_id FROM ancestry a
JOIN gfs_resources p ON p.resource_id=a.parent_resource_id
WHERE NOT p.resource_id=ANY(a.visited))
SELECT count(*) violations FROM ancestry a JOIN gfs_resources p
ON p.resource_id=a.parent_resource_id WHERE p.deleted_at IS NOT NULL`)
expect(Number(result.rows[0]?.violations)).toBe(0)
}
describeRealPostgres('GFS Phase 0 real PostgreSQL integrity', () => {
const database = `gfs_phase0_${randomBytes(6).toString('hex')}`
const connectionString = databaseUrl(
adminUrl ?? 'postgresql://[email protected]/postgres',
database
)
let adminPool: Pool
let pool: Pool
beforeAll(async () => {
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
pool = new Pool({ connectionString })
await initDb({ connect: () => pool.connect() })
}, 60_000)
afterAll(async () => {
try {
await endPoolAndWaitForClients(pool)
if (!adminPool) return
await adminPool.query(
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname = $1 AND pid <> pg_backend_pid()`,
[database]
)
await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
} finally {
await adminPool?.end()
}
})
it('serializes create/delete and makes delete observe the committed child', async () => {
const tree = await seedTree(pool, 'create-delete')
const creator = await pool.connect()
const deleter = await pool.connect()
try {
await Promise.all([begin(creator), begin(deleter)])
const creatorPid = Number((await creator.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
const deleterPid = Number((await deleter.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
await writer(creator).create({
drive: tree.drive,
parentId: tree.destinationA,
name: 'concurrent',
kind: 'directory',
})
const deletion = writer(deleter)
.delete({
drive: tree.drive,
resourceId: tree.destinationA,
ifMatch: 0,
})
.catch(error => error as { code?: string })
await waitUntilBlocked(pool, deleterPid, creatorPid, 'create/delete')
await creator.query('COMMIT')
await expect(deletion).resolves.toMatchObject({ code: 'not_empty' })
await deleter.query('ROLLBACK')
} finally {
await creator.query('ROLLBACK').catch(() => undefined)
await deleter.query('ROLLBACK').catch(() => undefined)
for (const client of [creator, deleter]) client.release()
}
await expectNoLiveNodeBelowTombstone(pool)
}, 20_000)
it('serializes move/delete and prevents a half-observed destination', async () => {
const tree = await seedTree(pool, 'move-delete')
const [mover, deleter] = await Promise.all([pool.connect(), pool.connect()])
try {
await Promise.all([begin(mover), begin(deleter)])
const moverPid = Number((await mover.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
const deleterPid = Number((await deleter.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
expect(await move(mover, tree, tree.destinationA)).toBe(1)
const deletion = writer(deleter)
.delete({
drive: tree.drive,
resourceId: tree.destinationA,
ifMatch: 0,
})
.catch(error => error as { code?: string })
await waitUntilBlocked(pool, deleterPid, moverPid, 'move/delete')
await mover.query('COMMIT')
await expect(deletion).resolves.toMatchObject({ code: 'not_empty' })
await deleter.query('ROLLBACK')
} finally {
await mover.query('ROLLBACK').catch(() => undefined)
await deleter.query('ROLLBACK').catch(() => undefined)
for (const client of [mover, deleter]) client.release()
}
await expectNoLiveNodeBelowTombstone(pool)
}, 20_000)
it('rejects moving a directory below its descendant', async () => {
const tree = await seedTree(pool, 'cycle')
const descendant = randomUUID()
await pool.query(
`INSERT INTO gfs_resources
(resource_id,drive,parent_resource_id,name,kind,path_cache,version,bytes)
VALUES ($1,$2,$3,'descendant','directory','/source/descendant',0,0)`,
[descendant, tree.drive, tree.source]
)
const client = await pool.connect()
try {
await begin(client)
await expect(
applyResourcePatch(
client,
operator,
tree.drive,
tree.source,
{ newParentId: descendant },
async () => undefined
)
).rejects.toMatchObject({ code: 'path_invalid' })
} finally {
await client.query('ROLLBACK').catch(() => undefined)
client.release()
}
const source = await pool.query(
'SELECT parent_resource_id,path_cache,version FROM gfs_resources WHERE resource_id=$1',
[tree.source]
)
expect(source.rows).toEqual([
{ parent_resource_id: tree.root, path_cache: '/source', version: 0 },
])
})
it('atomically refreshes path_cache for every node in a moved subtree', async () => {
const tree = await seedTree(pool, 'subtree-paths')
const [nested, leaf] = [randomUUID(), randomUUID()]
await pool.query(
`INSERT INTO gfs_resources
(resource_id, drive, parent_resource_id, name, kind, path_cache, version, bytes)
VALUES ($1,$3,$4,'nested','directory','/source/nested',0,0),
($2,$3,$1,'leaf.txt','file','/source/nested/leaf.txt',0,0)`,
[nested, leaf, tree.drive, tree.source]
)
const client = await pool.connect()
try {
await begin(client)
await applyResourcePatch(
client,
operator,
tree.drive,
tree.source,
{ newParentId: tree.destinationA },
async () => undefined
)
const beforeCommit = await pool.query(
`SELECT path_cache FROM gfs_resources
WHERE resource_id=ANY($1::uuid[]) ORDER BY path_cache`,
[[tree.source, tree.child, nested, leaf]]
)
expect(beforeCommit.rows.map(row => row.path_cache)).toEqual([
'/source',
'/source/child.txt',
'/source/nested',
'/source/nested/leaf.txt',
])
await client.query('COMMIT')
} finally {
await client.query('ROLLBACK').catch(() => undefined)
client.release()
}
const paths = await pool.query(
`SELECT name,path_cache FROM gfs_resources
WHERE resource_id=ANY($1::uuid[]) ORDER BY path_cache`,
[[tree.source, tree.child, nested, leaf]]
)
expect(paths.rows).toEqual([
{ name: 'source', path_cache: '/destination-a/source' },
{ name: 'child.txt', path_cache: '/destination-a/source/child.txt' },
{ name: 'nested', path_cache: '/destination-a/source/nested' },
{ name: 'leaf.txt', path_cache: '/destination-a/source/nested/leaf.txt' },
])
})
it('serializes two moves and leaves one coherent final parent and path', async () => {
const tree = await seedTree(pool, 'move-move')
const [first, second] = await Promise.all([pool.connect(), pool.connect()])
try {
await Promise.all([begin(first), begin(second)])
const firstPid = Number((await first.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
const secondPid = Number((await second.query('SELECT pg_backend_pid() pid')).rows[0]?.pid)
expect(await move(first, tree, tree.destinationA)).toBe(1)
const secondMove = move(second, tree, tree.destinationB)
await waitUntilBlocked(pool, secondPid, firstPid, 'move/move')
await first.query('COMMIT')
await expect(secondMove).resolves.toBe(2)
await second.query('COMMIT')
const final = await pool.query(
`SELECT parent_resource_id,path_cache,version FROM gfs_resources WHERE resource_id=$1`,
[tree.child]
)
expect(final.rows).toEqual([
{
parent_resource_id: tree.destinationB,
path_cache: '/destination-b/child.txt',
version: 2,
},
])
} finally {
await first.query('ROLLBACK').catch(() => undefined)
await second.query('ROLLBACK').catch(() => undefined)
for (const client of [first, second]) client.release()
}
}, 20_000)
it('never lets cleanup claim a generation while publication holds its row lock', async () => {
const tree = await seedTree(pool, 'publish-cleanup')
const [publisher, cleanup] = await Promise.all([pool.connect(), pool.connect()])
const paused = deferred()
const resume = deferred()
try {
await begin(cleanup)
const transactor: Transactor = {
transaction: async work => {
await begin(publisher)
const result = await work(publisher)
paused.resolve()
await resume.promise
await publisher.query('COMMIT')
return result
},
}
const manifests = new PgBlobStagingStore(pool)
const replacement = new GfsWriteService(transactor, blobs, manifests).replace({
drive: tree.drive,
resourceId: tree.child,
ifMatch: 0,
content: Buffer.from('replacement'),
})
await paused.promise
const store = new PgBlobStagingStore(cleanup)
await expect(store.claimExpiredCandidate(0)).resolves.toBeNull()
resume.resolve()
const published = await replacement
const row = await pool.query<{ blob_key: string | null }>(
'SELECT blob_key FROM gfs_resources WHERE resource_id=$1',
[tree.child]
)
expect(row.rows[0]?.blob_key).toBe(published.blobKey)
await cleanup.query('ROLLBACK')
} finally {
await publisher.query('ROLLBACK').catch(() => undefined)
await cleanup.query('ROLLBACK').catch(() => undefined)
for (const client of [publisher, cleanup]) client.release()
}
}, 20_000)
it('advances updatedAt after replace and move without changing an untouched parent', async () => {
const tree = await seedTree(pool, 'updated-at')
const seed = '2000-01-01T00:00:00.000Z'
await pool.query(
`UPDATE gfs_resources SET created_at = $2::timestamptz, updated_at = $2::timestamptz WHERE drive = $1`,
[tree.drive, seed]
)
const replaceClient = await pool.connect()
try {
const published = await writer(replaceClient).replace({
drive: tree.drive,
resourceId: tree.child,
ifMatch: 0,
content: Buffer.from('replacement'),
})
expect(published.updatedAt).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/)
expect(new Date(published.updatedAt).getTime()).toBeGreaterThan(new Date(seed).getTime())
} finally {
replaceClient.release()
}
const afterReplace = await pool.query<{
id: string
updated_at: Date
updated_at_text: string
}>(
`SELECT resource_id::text AS id, updated_at, updated_at::text AS updated_at_text
FROM gfs_resources WHERE drive = $1`,
[tree.drive]
)
const byId = Object.fromEntries(
afterReplace.rows.map(row => [row.id, row.updated_at.toISOString()])
)
expect(new Date(byId[tree.child] ?? 0).getTime()).toBeGreaterThan(new Date(seed).getTime())
expect(byId[tree.source]).toBe(new Date(seed).toISOString())
const beforeMoveTimestamp = afterReplace.rows.find(
row => row.id === tree.child
)?.updated_at_text
expect(beforeMoveTimestamp).toBeDefined()
const moveClient = await pool.connect()
try {
await begin(moveClient)
await move(moveClient, tree, tree.destinationA)
await moveClient.query('COMMIT')
} finally {
moveClient.release()
}
const afterMove = await pool.query<{ child_advanced: boolean; parent_at: Date }>(
`SELECT
(SELECT updated_at > $3::timestamptz FROM gfs_resources WHERE resource_id = $1)
AS child_advanced,
(SELECT updated_at FROM gfs_resources WHERE resource_id = $2) AS parent_at`,
[tree.child, tree.source, beforeMoveTimestamp]
)
// PostgreSQL keeps microseconds; JS Date/ISO output keeps only milliseconds.
// Compare at storage precision so two real updates in one millisecond still
// prove advancement, while an unchanged timestamp continues to fail.
expect(afterMove.rows[0]?.child_advanced).toBe(true)
expect(afterMove.rows[0]?.parent_at.toISOString()).toBe(new Date(seed).toISOString())
}, 20_000)
})