Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): clean superseded GFS fixture blobs
Alfredo Lopez065dacba3537
diff --git a/control-ui/lib/__tests__/gfsFixtureCleanup.test.ts b/control-ui/lib/__tests__/gfsFixtureCleanup.test.tsnew file mode 100644index 000000000..54080f5f3--- /dev/null+++ b/control-ui/lib/__tests__/gfsFixtureCleanup.test.ts@@ -0,0 +1,162 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import { assertGfsFixtureCleaned } from '../../../tests/e2e/gfsFixtureCleanupAssertions'+import { cleanupGfsFixture } from '../../../tests/e2e/gfsResourceFixtures'++const { gfsWriterPodMock, kubectlOutMock, runControlPostgresSqlMock } = vi.hoisted(() => ({+ gfsWriterPodMock: vi.fn(() => 'gfsc-writer-test'),+ kubectlOutMock: vi.fn(() => ''),+ runControlPostgresSqlMock: vi.fn(),+}))++vi.mock('../../../tests/e2e/gfsFixtureCore', async importOriginal => ({+ ...(await importOriginal<typeof import('../../../tests/e2e/gfsFixtureCore')>()),+ gfsWriterPod: gfsWriterPodMock,+ kubectlOut: kubectlOutMock,+ runControlPostgresSql: runControlPostgresSqlMock,+}))++const FIXTURE = 'e2e-gfs-v2-replace-cleanup'+const RESOURCE_ID = '70243f08-ba1d-45a6-b341-138a05bde79d'+const RESOURCE_RID = RESOURCE_ID.replaceAll('-', '')+const GENERATION = 'c85020b1-1111-4111-8111-111111111111'+const CURRENT_BLOB_KEY = `${RESOURCE_RID}/${GENERATION}`+const SUPERSEDED_GENERATION = 'd95020b1-2222-4222-8222-222222222222'+const SUPERSEDED_BLOB_KEY = `${RESOURCE_RID}/${SUPERSEDED_GENERATION}`++describe('GFS E2E fixture blob cleanup', () => {+ beforeEach(() => {+ gfsWriterPodMock.mockClear()+ kubectlOutMock.mockReset().mockReturnValue('')+ runControlPostgresSqlMock.mockReset()+ })++ it('removes the published generation and its superseded legacy flat blob before dropping their manifests', () => {+ runControlPostgresSqlMock+ .mockReturnValueOnce(+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${RESOURCE_RID}|legacy_flat|deleting\n`+ )+ .mockReturnValueOnce('')++ cleanupGfsFixture(FIXTURE)++ expect(kubectlOutMock).toHaveBeenCalledWith(+ [+ '-n',+ 'gfs',+ 'exec',+ 'gfsc-writer-test',+ '--',+ 'rm',+ '-f',+ `/data/gfs/.generations/${CURRENT_BLOB_KEY}`,+ ],+ 20_000+ )+ expect(kubectlOutMock).toHaveBeenCalledWith(+ ['-n', 'gfs', 'exec', 'gfsc-writer-test', '--', 'rm', '-f', `/data/gfs/${RESOURCE_RID}`],+ 20_000+ )+ const mutationSql = String(runControlPostgresSqlMock.mock.calls[1]?.[0] ?? '')+ expect(mutationSql).toContain('BEGIN;')+ expect(mutationSql).toContain('DELETE FROM gfs_blob_manifests')+ expect(mutationSql).toContain("WHERE state = 'deleting'")+ expect(mutationSql).toContain(`'${RESOURCE_RID}'`)+ expect(mutationSql).toContain('UPDATE gfs_resources')+ expect(mutationSql).toContain('COMMIT;')+ })++ it('rejects a generation key owned by another resource before deleting bytes or metadata', () => {+ const foreignKey = `${'a'.repeat(32)}/${GENERATION}`+ runControlPostgresSqlMock.mockReturnValue(`${RESOURCE_ID}|${foreignKey}|||\n`)++ expect(() => cleanupGfsFixture(FIXTURE)).toThrow('does not belong to resource')+ expect(kubectlOutMock).not.toHaveBeenCalled()+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()+ })++ it('removes an exact superseded immutable generation recorded by the fixture ledger', () => {+ runControlPostgresSqlMock+ .mockReturnValueOnce(+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${SUPERSEDED_BLOB_KEY}|generation|deleting\n`+ )+ .mockReturnValueOnce('')++ cleanupGfsFixture(FIXTURE)++ expect(kubectlOutMock).toHaveBeenCalledWith(+ [+ '-n',+ 'gfs',+ 'exec',+ 'gfsc-writer-test',+ '--',+ 'rm',+ '-f',+ `/data/gfs/.generations/${SUPERSEDED_BLOB_KEY}`,+ ],+ 20_000+ )+ const mutationSql = String(runControlPostgresSqlMock.mock.calls[1]?.[0] ?? '')+ expect(mutationSql).toContain(`'${SUPERSEDED_BLOB_KEY}'`)+ })++ it('never races production by deleting a staged or committed manifest', () => {+ runControlPostgresSqlMock.mockReturnValue(+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${CURRENT_BLOB_KEY}|generation|staged\n`+ )++ expect(() => cleanupGfsFixture(FIXTURE)).toThrow('invalid GFS fixture blob manifest row')+ expect(kubectlOutMock).not.toHaveBeenCalled()+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()+ })++ it('retains fixture metadata when an exact physical cleanup fails', () => {+ runControlPostgresSqlMock.mockReturnValue(+ `${RESOURCE_ID}|${CURRENT_BLOB_KEY}|${RESOURCE_RID}|legacy_flat|deleting\n`+ )+ kubectlOutMock.mockImplementationOnce(() => {+ throw new Error('simulated unlink failure')+ })++ expect(() => cleanupGfsFixture(FIXTURE)).toThrow(`failed to fully clean GFS fixture ${FIXTURE}`)+ expect(runControlPostgresSqlMock).toHaveBeenCalledOnce()+ })++ it('proves fixture manifests, legacy bytes, and the generation directory are all absent', () => {+ runControlPostgresSqlMock.mockReturnValue(`0|0|0|0|${RESOURCE_ID}\n`)++ assertGfsFixtureCleaned(FIXTURE)++ expect(kubectlOutMock).toHaveBeenCalledWith([+ '-n',+ 'gfs',+ 'exec',+ 'gfsc-writer-test',+ '--',+ 'test',+ '!',+ '-e',+ `/data/gfs/${RESOURCE_RID}`,+ ])+ expect(kubectlOutMock).toHaveBeenCalledWith([+ '-n',+ 'gfs',+ 'exec',+ 'gfsc-writer-test',+ '--',+ 'test',+ '!',+ '-e',+ `/data/gfs/.generations/${RESOURCE_RID}`,+ ])+ })++ it('rejects leftover blob-manifest authority even when resources are tombstoned', () => {+ runControlPostgresSqlMock.mockReturnValue(`0|0|0|1|${RESOURCE_ID}\n`)++ expect(() => assertGfsFixtureCleaned(FIXTURE)).toThrow(+ `GFS fixture ${FIXTURE} leaked active resources, authority, or blob manifests: 0|0|0|1`+ )+ expect(kubectlOutMock).not.toHaveBeenCalled()+ })+})diff --git a/tests/e2e/gfsFixtureCleanupAssertions.ts b/tests/e2e/gfsFixtureCleanupAssertions.tsindex 907ef3371..7bcb3fa2b 100644--- a/tests/e2e/gfsFixtureCleanupAssertions.ts+++ b/tests/e2e/gfsFixtureCleanupAssertions.ts@@ -27,17 +27,21 @@ export function assertGfsFixtureCleaned(name: string): void {WHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||(SELECT COUNT(*) FROM gfs_sharesWHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||+ (SELECT COUNT(*) FROM gfs_blob_manifests+ WHERE resource_id IN (SELECT resource_id FROM fixture_resources))::text || '|' ||COALESCE(array_to_string(array_agg(resource_id::text) FILTER (WHERE kind = 'file'), ','), '')FROM fixture_resources;`))- const [activeResources, grants, shares, fileIds = ''] = splitSqlRow(row)- if (activeResources !== '0' || grants !== '0' || shares !== '0') {+ const [activeResources, grants, shares, blobManifests, fileIds = ''] = splitSqlRow(row)+ if (activeResources !== '0' || grants !== '0' || shares !== '0' || blobManifests !== '0') {throw new Error(- `GFS fixture ${name} leaked active resources or authority rows: ${activeResources}|${grants}|${shares}`+ `GFS fixture ${name} leaked active resources, authority, or blob manifests: ${activeResources}|${grants}|${shares}|${blobManifests}`)}for (const resourceId of fileIds.split(',').filter(value => UUID_RE.test(value))) {+ const rid = ridOf(resourceId)+ kubectlOut(['-n', 'gfs', 'exec', gfsWriterPod(), '--', 'test', '!', '-e', `/data/gfs/${rid}`])kubectlOut(['-n','gfs',@@ -47,7 +51,7 @@ export function assertGfsFixtureCleaned(name: string): void {'test','!','-e',- `/data/gfs/${ridOf(resourceId)}`,+ `/data/gfs/.generations/${rid}`,])}}diff --git a/tests/e2e/gfsResourceFixtures.ts b/tests/e2e/gfsResourceFixtures.tsindex 6f97f4832..c45cce7a6 100644--- a/tests/e2e/gfsResourceFixtures.ts+++ b/tests/e2e/gfsResourceFixtures.ts@@ -41,18 +41,110 @@ function seedGfsBlob(resourceId: string, content: Buffer): void {}}+function normalizedGenerationBlobKey(resourceId: string, blobKey: string): string {+ const match =+ /^([0-9a-f]{32})\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i.exec(+ blobKey+ )+ if (!match) throw new Error(`refusing to clean invalid GFS generation key: ${blobKey}`)+ const ownerRid = match[1]!.toLowerCase()+ const expectedRid = ridOf(resourceId)+ if (ownerRid !== expectedRid) {+ throw new Error(`GFS generation key ${blobKey} does not belong to resource ${resourceId}`)+ }+ return `${ownerRid}/${match[2]!.toLowerCase()}`+}+function cleanupGfsBlob(resourceId: string, blobKey?: string | null): void {const physicalPath = blobKey- ? (() => {- if (!/^[0-9a-f]{32}\/[0-9a-f-]{36}$/i.test(blobKey)) {- throw new Error(`refusing to clean invalid GFS generation key: ${blobKey}`)- }- return `/data/gfs/.generations/${blobKey}`- })()+ ? `/data/gfs/.generations/${normalizedGenerationBlobKey(resourceId, blobKey)}`: `/data/gfs/${ridOf(resourceId)}`kubectlOut(['-n', 'gfs', 'exec', gfsWriterPod(), '--', 'rm', '-f', physicalPath], 20_000)}+function cleanupGfsGenerationDirectory(resourceId: string): void {+ const directory = `/data/gfs/.generations/${ridOf(resourceId)}`+ kubectlOut(+ [+ '-n',+ 'gfs',+ 'exec',+ gfsWriterPod(),+ '--',+ 'sh',+ '-c',+ `rmdir ${directory} 2>/dev/null || true`,+ ],+ 20_000+ )+}++interface GfsFixtureBlobInventoryRow {+ resourceId: string+ currentBlobKey: string | null+ manifestBlobKey: string | null+ manifestKind: 'generation' | 'legacy_flat' | null+ manifestState: 'deleting' | null+}++function fixtureBlobInventory(name: string): GfsFixtureBlobInventoryRow[] {+ const output = runControlPostgresSql(`+ SELECT resource.resource_id::text,+ coalesce(resource.blob_key, ''),+ coalesce(manifest.blob_key, ''),+ coalesce(manifest.candidate_kind, ''),+ coalesce(manifest.state, '')+ FROM gfs_resources AS resource+ LEFT JOIN gfs_blob_manifests AS manifest+ ON manifest.resource_id = resource.resource_id+ WHERE resource.drive = ${sqlLiteral(GFS_E2E_DRIVE)}+ AND resource.kind = 'file'+ AND (+ resource.path_cache = ${sqlLiteral(`/${name}`)}+ OR resource.path_cache LIKE ${sqlLiteral(`/${name}/%`)}+ )+ ORDER BY resource.resource_id, manifest.blob_key;+ `)+ return output+ .split('\n')+ .map(line => line.trim())+ .filter(Boolean)+ .map(line => {+ const parts = splitSqlRow(line)+ if (parts.length !== 5 || !UUID_RE.test(parts[0] ?? '')) {+ throw new Error(`invalid GFS fixture blob inventory row: ${line}`)+ }+ const [resourceId, rawCurrent = '', rawManifest = '', rawKind = '', rawState = ''] = parts+ const currentBlobKey = rawCurrent ? normalizedGenerationBlobKey(resourceId, rawCurrent) : null+ let manifestBlobKey: string | null = null+ let manifestKind: GfsFixtureBlobInventoryRow['manifestKind'] = null+ let manifestState: GfsFixtureBlobInventoryRow['manifestState'] = null+ if (rawManifest || rawKind || rawState) {+ if (+ !rawManifest ||+ (rawKind !== 'generation' && rawKind !== 'legacy_flat') ||+ rawState !== 'deleting'+ ) {+ throw new Error(`invalid GFS fixture blob manifest row: ${line}`)+ }+ manifestKind = rawKind+ manifestState = rawState+ if (manifestKind === 'generation') {+ manifestBlobKey = normalizedGenerationBlobKey(resourceId, rawManifest)+ } else {+ const expectedLegacyKey = ridOf(resourceId)+ if (rawManifest.toLowerCase() !== expectedLegacyKey) {+ throw new Error(+ `GFS legacy blob key ${rawManifest} does not belong to resource ${resourceId}`+ )+ }+ manifestBlobKey = expectedLegacyKey+ }+ }+ return { resourceId, currentBlobKey, manifestBlobKey, manifestKind, manifestState }+ })+}+export function seedGfsDirectoryFixture(name: string): GfsDirectoryFixture {assertFixtureName(name)const childName = `${name}-child`@@ -335,30 +427,65 @@ export function countGfsLiveChildrenByName(input: {export function cleanupGfsFixture(name: string): void {assertFixtureName(name)- const resources = runControlPostgresSql(`- SELECT resource_id::text || '|' || coalesce(blob_key, '')- FROM gfs_resources- WHERE drive = ${sqlLiteral(GFS_E2E_DRIVE)}- AND kind = 'file'- AND (path_cache = ${sqlLiteral(`/${name}`)} OR path_cache LIKE ${sqlLiteral(`/${name}/%`)});- `)- .split('\n')- .map(line => line.trim())- .filter(Boolean)- .map(splitSqlRow)- .filter(- (parts): parts is [string, string] => parts.length === 2 && UUID_RE.test(parts[0] ?? '')- )+ const inventory = fixtureBlobInventory(name)+ const blobTargets = new Map<string, { resourceId: string; blobKey: string | null }>()+ const deletingManifestKeys = new Set<string>()+ const resourceIds = new Set<string>()+ for (const row of inventory) {+ resourceIds.add(row.resourceId)+ // A fixture may start as a directly seeded legacy-flat blob and later+ // publish an immutable generation. Once blob_key points at the generation,+ // the original flat bytes are no longer visible through GFS but remain+ // protected by the production cleanup safety window. The fixture owns both+ // exact paths and must remove both instead of lowering that safety window.+ blobTargets.set(`${row.resourceId}|legacy_flat`, {+ resourceId: row.resourceId,+ blobKey: null,+ })+ if (row.currentBlobKey) {+ blobTargets.set(`${row.resourceId}|${row.currentBlobKey}`, {+ resourceId: row.resourceId,+ blobKey: row.currentBlobKey,+ })+ }+ if (row.manifestKind === 'generation' && row.manifestBlobKey) {+ blobTargets.set(`${row.resourceId}|${row.manifestBlobKey}`, {+ resourceId: row.resourceId,+ blobKey: row.manifestBlobKey,+ })+ }+ if (row.manifestState === 'deleting' && row.manifestBlobKey) {+ deletingManifestKeys.add(row.manifestBlobKey)+ }+ }const cleanupErrors: unknown[] = []- for (const [resourceId, blobKey] of resources) {+ for (const { resourceId, blobKey } of blobTargets.values()) {try {cleanupGfsBlob(resourceId, blobKey || null)} catch (error) {cleanupErrors.push(error)}}+ for (const resourceId of resourceIds) {+ try {+ cleanupGfsGenerationDirectory(resourceId)+ } catch (error) {+ cleanupErrors.push(error)+ }+ }+ if (cleanupErrors.length > 0) {+ throw new AggregateError(cleanupErrors, `failed to fully clean GFS fixture ${name}`)+ }try {+ const manifestKeys = [...deletingManifestKeys]+ const manifestCleanup = manifestKeys.length+ ? `DELETE FROM gfs_blob_manifests+ WHERE state = 'deleting'+ AND blob_key IN (${manifestKeys.map(sqlLiteral).join(', ')});`+ : ''runControlPostgresSql(`+ BEGIN;+WITH fixture_resources AS (SELECT resource_idFROM gfs_resources@@ -375,10 +502,14 @@ export function cleanupGfsFixture(name: string): void {)DELETE FROM gfs_shares WHERE resource_id IN (SELECT resource_id FROM fixture_resources);+ ${manifestCleanup}+UPDATE gfs_resourcesSET deleted_at = now(), updated_at = now()WHERE drive = ${sqlLiteral(GFS_E2E_DRIVE)}AND (path_cache = ${sqlLiteral(`/${name}`)} OR path_cache LIKE ${sqlLiteral(`/${name}/%`)});++ COMMIT;`)} catch (error) {cleanupErrors.push(error)