Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): clean superseded GFS fixture blobs

Alfredo Lopez065dacba3537
diff --git a/control-ui/lib/__tests__/gfsFixtureCleanup.test.ts b/control-ui/lib/__tests__/gfsFixtureCleanup.test.ts
new file mode 100644
index 000000000..54080f5f3
--- /dev/null
+++ b/control-ui/lib/__tests__/gfsFixtureCleanup.test.ts
@@ -0,0 +1,162 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { assertGfsFixtureCleaned } from '../../../tests/e2e/gfsFixtureCleanupAssertions'
+import { cleanupGfsFixture } from '../../../tests/e2e/gfsResourceFixtures'
+
+const { gfsWriterPodMock, kubectlOutMock, runControlPostgresSqlMock } = vi.hoisted(() => ({
+ gfsWriterPodMock: vi.fn(() => 'gfsc-writer-test'),
+ kubectlOutMock: vi.fn(() => ''),
+ runControlPostgresSqlMock: vi.fn(),
+}))
+
+vi.mock('../../../tests/e2e/gfsFixtureCore', async importOriginal => ({
+ ...(await importOriginal<typeof import('../../../tests/e2e/gfsFixtureCore')>()),
+ gfsWriterPod: gfsWriterPodMock,
+ kubectlOut: kubectlOutMock,
+ runControlPostgresSql: runControlPostgresSqlMock,
+}))
+
+const FIXTURE = 'e2e-gfs-v2-replace-cleanup'
+const RESOURCE_ID = '70243f08-ba1d-45a6-b341-138a05bde79d'
+const RESOURCE_RID = RESOURCE_ID.replaceAll('-', '')
+const GENERATION = 'c85020b1-1111-4111-8111-111111111111'
+const CURRENT_BLOB_KEY = `${RESOURCE_RID}/${GENERATION}`
+const SUPERSEDED_GENERATION = 'd95020b1-2222-4222-8222-222222222222'
+const SUPERSEDED_BLOB_KEY = `${RESOURCE_RID}/${SUPERSEDED_GENERATION}`
+
+describe('GFS E2E fixture blob cleanup', () => {
+ beforeEach(() => {
+ gfsWriterPodMock.mockClear()
+ kubectlOutMock.mockReset().mockReturnValue('')
+ runControlPostgresSqlMock.mockReset()
+ })
+
+ it('removes the published generation and its superseded legacy flat blob before dropping their manifests', () => {
+ runControlPostgresSqlMock
+ .mockReturnValueOnce(
+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${RESOURCE_RID}|legacy_flat|deleting\n`
+ )
+ .mockReturnValueOnce('')
+
+ cleanupGfsFixture(FIXTURE)
+
+ expect(kubectlOutMock).toHaveBeenCalledWith(
+ [
+ '-n',
+ 'gfs',
+ 'exec',
+ 'gfsc-writer-test',
+ '--',
+ 'rm',
+ '-f',
+ `/data/gfs/.generations/${CURRENT_BLOB_KEY}`,
+ ],
+ 20_000
+ )
+ expect(kubectlOutMock).toHaveBeenCalledWith(
+ ['-n', 'gfs', 'exec', 'gfsc-writer-test', '--', 'rm', '-f', `/data/gfs/${RESOURCE_RID}`],
+ 20_000
+ )
+ const mutationSql = String(runControlPostgresSqlMock.mock.calls[1]?.[0] ?? '')
+ expect(mutationSql).toContain('BEGIN;')
+ expect(mutationSql).toContain('DELETE FROM gfs_blob_manifests')
+ expect(mutationSql).toContain("WHERE state = 'deleting'")
+ expect(mutationSql).toContain(`'${RESOURCE_RID}'`)
+ expect(mutationSql).toContain('UPDATE gfs_resources')
+ expect(mutationSql).toContain('COMMIT;')
+ })
+
+ it('rejects a generation key owned by another resource before deleting bytes or metadata', () => {
+ const foreignKey = `${'a'.repeat(32)}/${GENERATION}`
+ runControlPostgresSqlMock.mockReturnValue(`${RESOURCE_ID}|${foreignKey}|||\n`)
+
+ expect(() => cleanupGfsFixture(FIXTURE)).toThrow('does not belong to resource')
+ expect(kubectlOutMock).not.toHaveBeenCalled()
+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()
+ })
+
+ it('removes an exact superseded immutable generation recorded by the fixture ledger', () => {
+ runControlPostgresSqlMock
+ .mockReturnValueOnce(
+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${SUPERSEDED_BLOB_KEY}|generation|deleting\n`
+ )
+ .mockReturnValueOnce('')
+
+ cleanupGfsFixture(FIXTURE)
+
+ expect(kubectlOutMock).toHaveBeenCalledWith(
+ [
+ '-n',
+ 'gfs',
+ 'exec',
+ 'gfsc-writer-test',
+ '--',
+ 'rm',
+ '-f',
+ `/data/gfs/.generations/${SUPERSEDED_BLOB_KEY}`,
+ ],
+ 20_000
+ )
+ const mutationSql = String(runControlPostgresSqlMock.mock.calls[1]?.[0] ?? '')
+ expect(mutationSql).toContain(`'${SUPERSEDED_BLOB_KEY}'`)
+ })
+
+ it('never races production by deleting a staged or committed manifest', () => {
+ runControlPostgresSqlMock.mockReturnValue(
+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${CURRENT_BLOB_KEY}|generation|staged\n`
+ )
+
+ expect(() => cleanupGfsFixture(FIXTURE)).toThrow('invalid GFS fixture blob manifest row')
+ expect(kubectlOutMock).not.toHaveBeenCalled()
+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()
+ })
+
+ it('retains fixture metadata when an exact physical cleanup fails', () => {
+ runControlPostgresSqlMock.mockReturnValue(
+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${RESOURCE_RID}|legacy_flat|deleting\n`
+ )
+ kubectlOutMock.mockImplementationOnce(() => {
+ throw new Error('simulated unlink failure')
+ })
+
+ expect(() => cleanupGfsFixture(FIXTURE)).toThrow(`failed to fully clean GFS fixture ${FIXTURE}`)
+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()
+ })
+
+ it('proves fixture manifests, legacy bytes, and the generation directory are all absent', () => {
+ runControlPostgresSqlMock.mockReturnValue(`0|0|0|0|${RESOURCE_ID}\n`)
+
+ assertGfsFixtureCleaned(FIXTURE)
+
+ expect(kubectlOutMock).toHaveBeenCalledWith([
+ '-n',
+ 'gfs',
+ 'exec',
+ 'gfsc-writer-test',
+ '--',
+ 'test',
+ '!',
+ '-e',
+ `/data/gfs/${RESOURCE_RID}`,
+ ])
+ expect(kubectlOutMock).toHaveBeenCalledWith([
+ '-n',
+ 'gfs',
+ 'exec',
+ 'gfsc-writer-test',
+ '--',
+ 'test',
+ '!',
+ '-e',
+ `/data/gfs/.generations/${RESOURCE_RID}`,
+ ])
+ })
+
+ it('rejects leftover blob-manifest authority even when resources are tombstoned', () => {
+ runControlPostgresSqlMock.mockReturnValue(`0|0|0|1|${RESOURCE_ID}\n`)
+
+ expect(() => assertGfsFixtureCleaned(FIXTURE)).toThrow(
+ `GFS fixture ${FIXTURE} leaked active resources, authority, or blob manifests: 0|0|0|1`
+ )
+ expect(kubectlOutMock).not.toHaveBeenCalled()
+ })
+})
diff --git a/tests/e2e/gfsFixtureCleanupAssertions.ts b/tests/e2e/gfsFixtureCleanupAssertions.ts
index 907ef3371..7bcb3fa2b 100644
--- a/tests/e2e/gfsFixtureCleanupAssertions.ts
+++ b/tests/e2e/gfsFixtureCleanupAssertions.ts
@@ -27,17 +27,21 @@ export function assertGfsFixtureCleaned(name: string): void {
WHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||
(SELECT COUNT(*) FROM gfs_shares
WHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||
+ (SELECT COUNT(*) FROM gfs_blob_manifests
+ WHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||
COALESCE(array_to_string(array_agg(resource_id::text) FILTER (WHERE kind = 'file'), ','), '')
FROM fixture_resources;
`)
)
- const [activeResources, grants, shares, fileIds = ''] = splitSqlRow(row)
- if (activeResources !== '0' || grants !== '0' || shares !== '0') {
+ const [activeResources, grants, shares, blobManifests, fileIds = ''] = splitSqlRow(row)
+ if (activeResources !== '0' || grants !== '0' || shares !== '0' || blobManifests !== '0') {
throw new Error(
- `GFS fixture ${name} leaked active resources or authority rows: ${activeResources}|${grants}|${shares}`
+ `GFS fixture ${name} leaked active resources, authority, or blob manifests: ${activeResources}|${grants}|${shares}|${blobManifests}`
)
}
for (const resourceId of fileIds.split(',').filter(value => UUID_RE.test(value))) {
+ const rid = ridOf(resourceId)
+ kubectlOut(['-n', 'gfs', 'exec', gfsWriterPod(), '--', 'test', '!', '-e', `/data/gfs/${rid}`])
kubectlOut([
'-n',
'gfs',
@@ -47,7 +51,7 @@ export function assertGfsFixtureCleaned(name: string): void {
'test',
'!',
'-e',
- `/data/gfs/${ridOf(resourceId)}`,
+ `/data/gfs/.generations/${rid}`,
])
}
}
diff --git a/tests/e2e/gfsResourceFixtures.ts b/tests/e2e/gfsResourceFixtures.ts
index 6f97f4832..c45cce7a6 100644
--- a/tests/e2e/gfsResourceFixtures.ts
+++ b/tests/e2e/gfsResourceFixtures.ts
@@ -41,18 +41,110 @@ function seedGfsBlob(resourceId: string, content: Buffer): void {
}
}
+function normalizedGenerationBlobKey(resourceId: string, blobKey: string): string {
+ const match =
+ /^([0-9a-f]{32})\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i.exec(
+ blobKey
+ )
+ if (!match) throw new Error(`refusing to clean invalid GFS generation key: ${blobKey}`)
+ const ownerRid = match[1]!.toLowerCase()
+ const expectedRid = ridOf(resourceId)
+ if (ownerRid !== expectedRid) {
+ throw new Error(`GFS generation key ${blobKey} does not belong to resource ${resourceId}`)
+ }
+ return `${ownerRid}/${match[2]!.toLowerCase()}`
+}
+
function cleanupGfsBlob(resourceId: string, blobKey?: string | null): void {
const physicalPath = blobKey
- ? (() => {
- if (!/^[0-9a-f]{32}\/[0-9a-f-]{36}$/i.test(blobKey)) {
- throw new Error(`refusing to clean invalid GFS generation key: ${blobKey}`)
- }
- return `/data/gfs/.generations/${blobKey}`
- })()
+ ? `/data/gfs/.generations/${normalizedGenerationBlobKey(resourceId, blobKey)}`
: `/data/gfs/${ridOf(resourceId)}`
kubectlOut(['-n', 'gfs', 'exec', gfsWriterPod(), '--', 'rm', '-f', physicalPath], 20_000)
}
+function cleanupGfsGenerationDirectory(resourceId: string): void {
+ const directory = `/data/gfs/.generations/${ridOf(resourceId)}`
+ kubectlOut(
+ [
+ '-n',
+ 'gfs',
+ 'exec',
+ gfsWriterPod(),
+ '--',
+ 'sh',
+ '-c',
+ `rmdir ${directory} 2>/dev/null || true`,
+ ],
+ 20_000
+ )
+}
+
+interface GfsFixtureBlobInventoryRow {
+ resourceId: string
+ currentBlobKey: string | null
+ manifestBlobKey: string | null
+ manifestKind: 'generation' | 'legacy_flat' | null
+ manifestState: 'deleting' | null
+}
+
+function fixtureBlobInventory(name: string): GfsFixtureBlobInventoryRow[] {
+ const output = runControlPostgresSql(`
+ SELECT resource.resource_id::text,
+ coalesce(resource.blob_key, ''),
+ coalesce(manifest.blob_key, ''),
+ coalesce(manifest.candidate_kind, ''),
+ coalesce(manifest.state, '')
+ FROM gfs_resources AS resource
+ LEFT JOIN gfs_blob_manifests AS manifest
+ ON manifest.resource_id = resource.resource_id
+ WHERE resource.drive = ${sqlLiteral(GFS_E2E_DRIVE)}
+ AND resource.kind = 'file'
+ AND (
+ resource.path_cache = ${sqlLiteral(`/${name}`)}
+ OR resource.path_cache LIKE ${sqlLiteral(`/${name}/%`)}
+ )
+ ORDER BY resource.resource_id, manifest.blob_key;
+ `)
+ return output
+ .split('\n')
+ .map(line => line.trim())
+ .filter(Boolean)
+ .map(line => {
+ const parts = splitSqlRow(line)
+ if (parts.length !== 5 || !UUID_RE.test(parts[0] ?? '')) {
+ throw new Error(`invalid GFS fixture blob inventory row: ${line}`)
+ }
+ const [resourceId, rawCurrent = '', rawManifest = '', rawKind = '', rawState = ''] = parts
+ const currentBlobKey = rawCurrent ? normalizedGenerationBlobKey(resourceId, rawCurrent) : null
+ let manifestBlobKey: string | null = null
+ let manifestKind: GfsFixtureBlobInventoryRow['manifestKind'] = null
+ let manifestState: GfsFixtureBlobInventoryRow['manifestState'] = null
+ if (rawManifest || rawKind || rawState) {
+ if (
+ !rawManifest ||
+ (rawKind !== 'generation' && rawKind !== 'legacy_flat') ||
+ rawState !== 'deleting'
+ ) {
+ throw new Error(`invalid GFS fixture blob manifest row: ${line}`)
+ }
+ manifestKind = rawKind
+ manifestState = rawState
+ if (manifestKind === 'generation') {
+ manifestBlobKey = normalizedGenerationBlobKey(resourceId, rawManifest)
+ } else {
+ const expectedLegacyKey = ridOf(resourceId)
+ if (rawManifest.toLowerCase() !== expectedLegacyKey) {
+ throw new Error(
+ `GFS legacy blob key ${rawManifest} does not belong to resource ${resourceId}`
+ )
+ }
+ manifestBlobKey = expectedLegacyKey
+ }
+ }
+ return { resourceId, currentBlobKey, manifestBlobKey, manifestKind, manifestState }
+ })
+}
+
export function seedGfsDirectoryFixture(name: string): GfsDirectoryFixture {
assertFixtureName(name)
const childName = `${name}-child`
@@ -335,30 +427,65 @@ export function countGfsLiveChildrenByName(input: {
export function cleanupGfsFixture(name: string): void {
assertFixtureName(name)
- const resources = runControlPostgresSql(`
- SELECT resource_id::text || '|' || coalesce(blob_key, '')
- FROM gfs_resources
- WHERE drive = ${sqlLiteral(GFS_E2E_DRIVE)}
- AND kind = 'file'
- AND (path_cache = ${sqlLiteral(`/${name}`)} OR path_cache LIKE ${sqlLiteral(`/${name}/%`)});
- `)
- .split('\n')
- .map(line => line.trim())
- .filter(Boolean)
- .map(splitSqlRow)
- .filter(
- (parts): parts is [string, string] => parts.length === 2 && UUID_RE.test(parts[0] ?? '')
- )
+ const inventory = fixtureBlobInventory(name)
+ const blobTargets = new Map<string, { resourceId: string; blobKey: string | null }>()
+ const deletingManifestKeys = new Set<string>()
+ const resourceIds = new Set<string>()
+ for (const row of inventory) {
+ resourceIds.add(row.resourceId)
+ // A fixture may start as a directly seeded legacy-flat blob and later
+ // publish an immutable generation. Once blob_key points at the generation,
+ // the original flat bytes are no longer visible through GFS but remain
+ // protected by the production cleanup safety window. The fixture owns both
+ // exact paths and must remove both instead of lowering that safety window.
+ blobTargets.set(`${row.resourceId}|legacy_flat`, {
+ resourceId: row.resourceId,
+ blobKey: null,
+ })
+ if (row.currentBlobKey) {
+ blobTargets.set(`${row.resourceId}|${row.currentBlobKey}`, {
+ resourceId: row.resourceId,
+ blobKey: row.currentBlobKey,
+ })
+ }
+ if (row.manifestKind === 'generation' && row.manifestBlobKey) {
+ blobTargets.set(`${row.resourceId}|${row.manifestBlobKey}`, {
+ resourceId: row.resourceId,
+ blobKey: row.manifestBlobKey,
+ })
+ }
+ if (row.manifestState === 'deleting' && row.manifestBlobKey) {
+ deletingManifestKeys.add(row.manifestBlobKey)
+ }
+ }
const cleanupErrors: unknown[] = []
- for (const [resourceId, blobKey] of resources) {
+ for (const { resourceId, blobKey } of blobTargets.values()) {
try {
cleanupGfsBlob(resourceId, blobKey || null)
} catch (error) {
cleanupErrors.push(error)
}
}
+ for (const resourceId of resourceIds) {
+ try {
+ cleanupGfsGenerationDirectory(resourceId)
+ } catch (error) {
+ cleanupErrors.push(error)
+ }
+ }
+ if (cleanupErrors.length > 0) {
+ throw new AggregateError(cleanupErrors, `failed to fully clean GFS fixture ${name}`)
+ }
try {
+ const manifestKeys = [...deletingManifestKeys]
+ const manifestCleanup = manifestKeys.length
+ ? `DELETE FROM gfs_blob_manifests
+ WHERE state = 'deleting'
+ AND blob_key IN (${manifestKeys.map(sqlLiteral).join(', ')});`
+ : ''
runControlPostgresSql(`
+ BEGIN;
+
WITH fixture_resources AS (
SELECT resource_id
FROM gfs_resources
@@ -375,10 +502,14 @@ export function cleanupGfsFixture(name: string): void {
)
DELETE FROM gfs_shares WHERE resource_id IN (SELECT resource_id FROM fixture_resources);
+ ${manifestCleanup}
+
UPDATE gfs_resources
SET deleted_at = now(), updated_at = now()
WHERE drive = ${sqlLiteral(GFS_E2E_DRIVE)}
AND (path_cache = ${sqlLiteral(`/${name}`)} OR path_cache LIKE ${sqlLiteral(`/${name}/%`)});
+
+ COMMIT;
`)
} catch (error) {
cleanupErrors.push(error)