Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(gfs): close reclaim and resume evidence gaps
Alfredo Lopezf2e374322438
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 135a06e4f..d6369407a 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.450",+ "version": "0.1.451","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.450",+ "version": "0.1.451","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex 4e55b4f60..a9613931e 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.450",+ "version": "0.1.451","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.tsindex 6e2ba875b..ec09b7c18 100644--- a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts+++ b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts@@ -221,6 +221,45 @@ describeRealPostgres('GFS upload finalizer on real PostgreSQL + BlobStore', () =expect(manifests.rows[0]?.count).toBe(0)}, 30_000)+ it('fences an old finalizer after reclaim so it cannot publish a resource or receipt', async () => {+ const { upload, uploadPart } = await seedUpload()+ await pool.query(+ `UPDATE gfs_upload_sessions+ SET session_epoch = session_epoch + 1,+ finalizing_started_at = now() - interval '1 hour'+ WHERE upload_id = $1`,+ [upload.uploadId]+ )++ await expect(finalizer(() => true)(upload, [uploadPart])).rejects.toMatchObject({+ code: 'upload_aborted',+ })++ const sessionRow = await pool.query(+ `SELECT state, session_epoch, result_resource_id, result_version, result_sha256+ FROM gfs_upload_sessions WHERE upload_id = $1`,+ [upload.uploadId]+ )+ expect(sessionRow.rows[0]).toMatchObject({+ state: 'finalizing',+ session_epoch: '1',+ result_resource_id: null,+ result_version: null,+ result_sha256: null,+ })++ const resource = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_resources WHERE resource_id = $1`,+ [upload.uploadId]+ )+ expect(resource.rows[0]?.count).toBe(0)+ const manifests = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,+ [upload.uploadId]+ )+ expect(manifests.rows[0]?.count).toBe(0)+ }, 30_000)+it('rolls back the real resource and receipt when authorization is denied', async () => {const { upload, uploadPart } = await seedUpload()await expect(finalizer(() => false)(upload, [uploadPart])).rejects.toMatchObject({diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.tsindex b528dd7d8..6b4783180 100644--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'import { join } from 'node:path'import { Readable } from 'node:stream'import { Pool } from 'pg'+import { GfsError } from '../../gfs-controller/src/api/errors.js'import type { GfsUploadConfig } from '../../gfs-controller/src/config.js'import { CommitOutcomeUnknownError, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'import type {@@ -513,6 +514,78 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {})})+ it('does not let a reclaimed finalizer clean the newer attempt parts', async () => {+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x85)+ const base = new GfsUploadSessionService({+ db: pool,+ tx: new PgTransactor(pool),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ })+ const created = await base.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '4'.repeat(32),+ name: 'reclaimed-finalizer.bin',+ sizeBytes: bytes.length,+ idempotencyKey: randomUUID(),+ })+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ await base.putPart(+ created.session.uploadId,+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),+ checksum(bytes),+ Readable.from([bytes]) as never,+ principal+ )+ const partRow = await pool.query<{ staging_path: string }>(+ `SELECT staging_path FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,+ [created.session.uploadId]+ )+ const stagedPath = String(partRow.rows[0]?.staging_path)++ const staleFinalizer = new GfsUploadSessionService({+ db: pool,+ tx: new PgTransactor(pool),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ finalize: async session => {+ await pool.query(+ `UPDATE gfs_upload_sessions+ SET state = 'uploading', session_epoch = session_epoch + 1,+ finalizing_started_at = NULL, expires_at = now() + interval '1 day'+ WHERE upload_id = $1 AND state = 'finalizing'`,+ [session.uploadId]+ )+ throw new GfsError('finalization_failed', 'synthetic reclaimed finalizer failure')+ },+ })++ await expect(+ staleFinalizer.complete(created.session.uploadId, principal)+ ).rejects.toMatchObject({ code: 'finalization_failed' })++ const sessionRow = await pool.query(+ `SELECT state, session_epoch, cleanup_at FROM gfs_upload_sessions WHERE upload_id = $1`,+ [created.session.uploadId]+ )+ expect(sessionRow.rows[0]).toMatchObject({ state: 'uploading', session_epoch: '1' })+ expect(sessionRow.rows[0]?.cleanup_at).toBeNull()+ expect(+ (+ await pool.query(+ `SELECT state FROM gfs_upload_parts WHERE upload_id = $1 AND part_number = 0`,+ [created.session.uploadId]+ )+ ).rows[0]+ ).toEqual({ state: 'committed' })+ await expect(stat(stagedPath)).resolves.toBeDefined()+ }, 30_000)+it('fences and cleans a stale reserved lease while its row lock is held', async () => {const created = await uploads.create({drive,@@ -743,13 +816,16 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {)await pool.query(`UPDATE gfs_upload_sessions- SET state = 'finalizing', finalizing_started_at = now() - interval '1 hour'+ SET state = 'finalizing',+ finalizing_started_at = now() - interval '1 hour',+ expires_at = now() - interval '1 second'WHERE upload_id = $1`,[created.session.uploadId])await uploads.reconcile()const row = await pool.query(- `SELECT state, session_epoch, finalizing_started_at FROM gfs_upload_sessions WHERE upload_id = $1`,+ `SELECT state, session_epoch, finalizing_started_at, expires_at+ FROM gfs_upload_sessions WHERE upload_id = $1`,[created.session.uploadId])expect(row.rows[0]).toMatchObject({@@ -757,6 +833,7 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {finalizing_started_at: null,})expect(Number(row.rows[0]?.session_epoch)).toBe(Number(before.rows[0]?.session_epoch) + 1)+ expect(new Date(String(row.rows[0]?.expires_at)).getTime()).toBeGreaterThan(Date.now())})it('rejects cancel during finalization and leaves one completed receipt', async () => {diff --git a/control-ui/e2e/gfs-upload-v2.spec.ts b/control-ui/e2e/gfs-upload-v2.spec.tsindex ac665c15d..31f8c5dda 100644--- a/control-ui/e2e/gfs-upload-v2.spec.ts+++ b/control-ui/e2e/gfs-upload-v2.spec.ts@@ -51,6 +51,44 @@ async function openWritableFolder(return current}+/**+ * Exercise the actual drop target while retaining Playwright's file-backed+ * File object. The capture listener runs before the React change handler clears+ * the native input, then the same file is delivered through a real DragEvent+ * sequence to the visible drop zone. This keeps the journey user-facing while+ * avoiding a synthetic upload request or direct storage mutation.+ */+async function dropFileIntoUploadDialog(+ dialog: import('@playwright/test').Locator,+ filePath: string+): Promise<void> {+ const input = dialog.getByLabel('Choose file to upload')+ const initialDropzone = dialog.locator('label').filter({ hasText: 'Drag and drop' })+ const labelFor = await initialDropzone.getAttribute('for')+ if (!labelFor) throw new Error('upload drop zone has no associated file input')+ await input.evaluate(element => {+ element.addEventListener(+ 'change',+ event => {+ const file = (event.currentTarget as HTMLInputElement).files?.[0]+ if (file) (window as Window & { __gfsDropFile?: File }).__gfsDropFile = file+ },+ { capture: true, once: true }+ )+ })+ await input.setInputFiles(filePath)+ await dialog.locator(`label[for="${labelFor}"]`).evaluate(element => {+ const file = (window as Window & { __gfsDropFile?: File }).__gfsDropFile+ if (!file) throw new Error('file capture failed before drag-and-drop')+ const dataTransfer = new DataTransfer()+ dataTransfer.items.add(file)+ for (const type of ['dragenter', 'dragover', 'drop']) {+ element.dispatchEvent(new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer }))+ }+ delete (window as Window & { __gfsDropFile?: File }).__gfsDropFile+ })+}+async function observeTwoProgressValues(progress: import('@playwright/test').Locator,timeout = 600_000@@ -216,6 +254,64 @@ test.describe('GFS Upload v2 — Control UI large-upload project', () => {await exerciseReplace(page, byteLength)})}++ test('resumes the same persisted session after a visible drag-and-drop pause and reload', async ({+ page,+ }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-drag-resume')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ const source = await createDiskUploadFixture(32 * 1024 * 1024, '.bin', fixtureName)+ try {+ await openWritableFolder(page, fixture.name)+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()+ let dialog = page.getByRole('dialog', { name: 'Upload file' })+ await dropFileIntoUploadDialog(dialog, source.filePath)+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()++ const progress = page.getByRole('progressbar', {+ name: `Upload progress for ${source.fileName}`,+ })+ await expect(progress).toBeVisible({ timeout: 60_000 })+ await dialog.getByRole('button', { name: 'Pause', exact: true }).click()+ await expect(dialog.getByRole('button', { name: 'Resume', exact: true })).toBeVisible({+ timeout: 60_000,+ })++ await page.reload()+ const resumedCurrent = await openWritableFolder(page, fixture.name)+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()+ dialog = page.getByRole('dialog', { name: 'Upload file' })+ const statusResponse = page.waitForResponse(+ response =>+ response.request().method() === 'GET' &&+ response.url().includes('/gfs/proxy/v1/uploads/') &&+ response.url().includes('/status')+ )+ await dropFileIntoUploadDialog(dialog, source.filePath)+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()+ expect((await statusResponse).status()).toBe(200)+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({+ timeout: 600_000,+ })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })+ await expect(+ resumedCurrent.getByRole('listitem').filter({ hasText: source.fileName })+ ).toHaveCount(1)+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })})test.describe('GFS Upload v2 — approved negative Control UI journeys', () => {diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.tsindex 6e07dd637..26cf90781 100644--- a/desktop-app/src/__tests__/appService.gfs.test.ts+++ b/desktop-app/src/__tests__/appService.gfs.test.ts@@ -411,16 +411,29 @@ describe('AppService GFS upload security scope', () => {}})- it('maps a persisted active record to the renderer uploading state after restart', async () => {+ it('maps a live persisted active record to the renderer uploading state', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-snapshot-state-'))try {const statePath = join(root, 'gfs-upload-sessions.json')const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))const service = authenticatedUploadService(statePath)+ const job = {+ snapshot: vi.fn(() => ({+ state: 'uploading',+ session: record.session,+ uploadedBytes: record.session.committedBytes,+ totalBytes: record.fileSize,+ })),+ }+ service.gfsUploadJobs.set(record.uploadId, {+ job,+ promise: Promise.resolve(record.session),+ scope: record.scope,+ })await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({- state: 'suspended_auth',+ state: 'uploading',uploadedBytes: 0,totalBytes: 4,})diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex c43c7c19e..deae760e7 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.74",+ "version": "0.1.75","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.74",+ "version": "0.1.75","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex 5acf68823..2b5f2fb80 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.74",+ "version": "0.1.75","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex 2fb0956e3..76a00b2e0 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -8,7 +8,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.7.0',- externalRestApiVersion: '0.1.74',+ externalRestApiVersion: '0.1.75',rpcProxyVersion: '0.1.64',desktopVersion: '0.7.0',minimumDesktopVersion: '0.1.252',diff --git a/external-rest-api/test/routes.gfs.test.ts b/external-rest-api/test/routes.gfs.test.tsindex 6c5e3a14e..0973698b8 100644--- a/external-rest-api/test/routes.gfs.test.ts+++ b/external-rest-api/test/routes.gfs.test.ts@@ -131,7 +131,7 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',expect(clientMock.controlApiStreamRequest).toHaveBeenCalledTimes(2)})- it('delegates missing and create-body-mismatched drive validation to control-api', async () => {+ it('keeps capabilities drive-independent while control-api rejects mismatched creates', async () => {clientMock.controlApiStreamRequest.mockImplementation(async (method: string,@@ -140,13 +140,13 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',) => {const drive = options.query?.driveconst body = typeof options.body === 'string' ? JSON.parse(options.body) : undefined- if (!drive) {- throw new ControlApiError('drive required', 400, { error: 'drive_required' })- }if (method === 'POST' && path === '/external/gfs/uploads' && body?.drive !== drive) {throw new ControlApiError('drive mismatch', 400, { error: 'drive_mismatch' })}- return new Response(JSON.stringify({ ok: true }), { status: 200 })+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ })})const missing = await request(buildApp())@@ -156,8 +156,8 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',.post('/me/gfs/uploads?drive=archive').set('authorization', 'Bearer sess-xyz').send({ drive: 'main', operation: 'create' })- expect(missing.status).toBe(400)- expect(missing.body).toEqual({ error: 'drive_required' })+ expect(missing.status).toBe(200)+ expect(missing.body).toEqual({ upload: { resumableV2: { enabled: true } } })expect(mismatched.status).toBe(400)expect(mismatched.body).toEqual({ error: 'drive_mismatch' })expect(clientMock.controlApiStreamRequest).toHaveBeenCalledTimes(2)diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex 3319633d8..d92b8d7ce 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -56,7 +56,7 @@ class MemoryDb {}if (sql.startsWith(- "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()"+ "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()")) {return { rows: [] }@@ -369,6 +369,16 @@ class MemoryDb {}if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {const row = this.sessions.find(session => session.upload_id === String(values[0]))!+ if (sql.includes("state = 'finalizing' AND session_epoch = $3")) {+ if (row.state !== 'finalizing' || Number(row.session_epoch) !== Number(values[2])) {+ return { rows: [] }+ }+ row.state = 'failed'+ row.failure_code = values[1]+ row.active_part_count = 0+ row.finalizing_started_at = null+ return { rows: [{ upload_id: row.upload_id }] }+ }row.state = 'failed'row.failure_code = values[1]row.active_part_count = 0diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex 6708af90f..a9d750ec3 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -1349,14 +1349,19 @@ export class GfsUploadSessionService {await client.query(`UPDATE gfs_upload_sessionsSET state = 'uploading', session_epoch = session_epoch + 1,- finalizing_started_at = NULL, updated_at = now()+ finalizing_started_at = NULL,+ expires_at = GREATEST(+ expires_at,+ now() + ($2::bigint * interval '1 second')+ ),+ updated_at = now()WHERE upload_id = $1 AND state = 'finalizing'`,- [uploadId]+ [uploadId, this.deps.config.sessionTtlSeconds])}const expired = await client.query(`SELECT upload_id FROM gfs_upload_sessions- WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()+ WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()LIMIT 100FOR UPDATE SKIP LOCKED`,[]@@ -1628,16 +1633,19 @@ export class GfsUploadSessionService {)} catch (error) {const failureCode = error instanceof GfsError ? error.code : 'finalization_failed'+ let ownsFailureCleanup = falseawait this.deps.tx.transaction(async client => {const locked = await this.lockSession(client, uploadId, principal)if (locked.state === 'finalizing' && locked.sessionEpoch === started.sessionEpoch) {- await client.query(+ const transitioned = await client.query(`UPDATE gfs_upload_sessionsSET state = 'failed', failure_code = $2, active_part_count = 0,finalizing_started_at = NULL, updated_at = now()- WHERE upload_id = $1`,- [uploadId, failureCode]+ WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $3+ RETURNING upload_id`,+ [uploadId, failureCode, started.sessionEpoch])+ ownsFailureCleanup = transitioned.rows.length === 1}})// `deps.finalize` has already settled, so this controller no longer@@ -1646,6 +1654,10 @@ export class GfsUploadSessionService {// until the part timeout, preventing the direct failure cleanup below.const current = this.inFlightFinalizers.get(uploadId)if (current?.controller === finalizationAbort) this.inFlightFinalizers.delete(uploadId)+ // A reclaimed epoch belongs to a newer attempt. The old finalizer may+ // report its own failure, but it must never delete shared parts or the+ // staging directory owned by the reclaimed session.+ if (!ownsFailureCleanup) throw errorawait this.waitForQuiescence(uploadId)if (!this.hasInFlight(uploadId) &&@@ -1653,10 +1665,19 @@ export class GfsUploadSessionService {) {await this.deps.tx.transaction(async client => {- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])await client.query(- `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`,- [uploadId]+ `DELETE FROM gfs_upload_parts+ WHERE upload_id = $1+ AND EXISTS (+ SELECT 1 FROM gfs_upload_sessions+ WHERE upload_id = $1 AND state = 'failed' AND session_epoch = $2+ )`,+ [uploadId, started.sessionEpoch]+ )+ await client.query(+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now()+ WHERE upload_id = $1 AND state = 'failed' AND session_epoch = $2`,+ [uploadId, started.sessionEpoch])}).catch(() => undefined)