Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(gfs): close reclaim and resume evidence gaps

Alfredo Lopezf2e374322438
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 135a06e4f..d6369407a 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.450",
+ "version": "0.1.451",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.450",
+ "version": "0.1.451",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 4e55b4f60..a9613931e 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.450",
+ "version": "0.1.451",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts
index 6e2ba875b..ec09b7c18 100644
--- a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts
@@ -221,6 +221,45 @@ describeRealPostgres('GFS upload finalizer on real PostgreSQL + BlobStore', () =
expect(manifests.rows[0]?.count).toBe(0)
}, 30_000)
+ it('fences an old finalizer after reclaim so it cannot publish a resource or receipt', async () => {
+ const { upload, uploadPart } = await seedUpload()
+ await pool.query(
+ `UPDATE gfs_upload_sessions
+ SET session_epoch = session_epoch + 1,
+ finalizing_started_at = now() - interval '1 hour'
+ WHERE upload_id = $1`,
+ [upload.uploadId]
+ )
+
+ await expect(finalizer(() => true)(upload, [uploadPart])).rejects.toMatchObject({
+ code: 'upload_aborted',
+ })
+
+ const sessionRow = await pool.query(
+ `SELECT state, session_epoch, result_resource_id, result_version, result_sha256
+ FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [upload.uploadId]
+ )
+ expect(sessionRow.rows[0]).toMatchObject({
+ state: 'finalizing',
+ session_epoch: '1',
+ result_resource_id: null,
+ result_version: null,
+ result_sha256: null,
+ })
+
+ const resource = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_resources WHERE resource_id = $1`,
+ [upload.uploadId]
+ )
+ expect(resource.rows[0]?.count).toBe(0)
+ const manifests = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,
+ [upload.uploadId]
+ )
+ expect(manifests.rows[0]?.count).toBe(0)
+ }, 30_000)
+
it('rolls back the real resource and receipt when authorization is denied', async () => {
const { upload, uploadPart } = await seedUpload()
await expect(finalizer(() => false)(upload, [uploadPart])).rejects.toMatchObject({
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
index b528dd7d8..6b4783180 100644
--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Readable } from 'node:stream'
import { Pool } from 'pg'
+import { GfsError } from '../../gfs-controller/src/api/errors.js'
import type { GfsUploadConfig } from '../../gfs-controller/src/config.js'
import { CommitOutcomeUnknownError, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'
import type {
@@ -513,6 +514,78 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
})
})
+ it('does not let a reclaimed finalizer clean the newer attempt parts', async () => {
+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x85)
+ const base = new GfsUploadSessionService({
+ db: pool,
+ tx: new PgTransactor(pool),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ })
+ const created = await base.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '4'.repeat(32),
+ name: 'reclaimed-finalizer.bin',
+ sizeBytes: bytes.length,
+ idempotencyKey: randomUUID(),
+ })
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ await base.putPart(
+ created.session.uploadId,
+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ principal
+ )
+ const partRow = await pool.query<{ staging_path: string }>(
+ `SELECT staging_path FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,
+ [created.session.uploadId]
+ )
+ const stagedPath = String(partRow.rows[0]?.staging_path)
+
+ const staleFinalizer = new GfsUploadSessionService({
+ db: pool,
+ tx: new PgTransactor(pool),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ finalize: async session => {
+ await pool.query(
+ `UPDATE gfs_upload_sessions
+ SET state = 'uploading', session_epoch = session_epoch + 1,
+ finalizing_started_at = NULL, expires_at = now() + interval '1 day'
+ WHERE upload_id = $1 AND state = 'finalizing'`,
+ [session.uploadId]
+ )
+ throw new GfsError('finalization_failed', 'synthetic reclaimed finalizer failure')
+ },
+ })
+
+ await expect(
+ staleFinalizer.complete(created.session.uploadId, principal)
+ ).rejects.toMatchObject({ code: 'finalization_failed' })
+
+ const sessionRow = await pool.query(
+ `SELECT state, session_epoch, cleanup_at FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ expect(sessionRow.rows[0]).toMatchObject({ state: 'uploading', session_epoch: '1' })
+ expect(sessionRow.rows[0]?.cleanup_at).toBeNull()
+ expect(
+ (
+ await pool.query(
+ `SELECT state FROM gfs_upload_parts WHERE upload_id = $1 AND part_number = 0`,
+ [created.session.uploadId]
+ )
+ ).rows[0]
+ ).toEqual({ state: 'committed' })
+ await expect(stat(stagedPath)).resolves.toBeDefined()
+ }, 30_000)
+
it('fences and cleans a stale reserved lease while its row lock is held', async () => {
const created = await uploads.create({
drive,
@@ -743,13 +816,16 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
)
await pool.query(
`UPDATE gfs_upload_sessions
- SET state = 'finalizing', finalizing_started_at = now() - interval '1 hour'
+ SET state = 'finalizing',
+ finalizing_started_at = now() - interval '1 hour',
+ expires_at = now() - interval '1 second'
WHERE upload_id = $1`,
[created.session.uploadId]
)
await uploads.reconcile()
const row = await pool.query(
- `SELECT state, session_epoch, finalizing_started_at FROM gfs_upload_sessions WHERE upload_id = $1`,
+ `SELECT state, session_epoch, finalizing_started_at, expires_at
+ FROM gfs_upload_sessions WHERE upload_id = $1`,
[created.session.uploadId]
)
expect(row.rows[0]).toMatchObject({
@@ -757,6 +833,7 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
finalizing_started_at: null,
})
expect(Number(row.rows[0]?.session_epoch)).toBe(Number(before.rows[0]?.session_epoch) + 1)
+ expect(new Date(String(row.rows[0]?.expires_at)).getTime()).toBeGreaterThan(Date.now())
})
it('rejects cancel during finalization and leaves one completed receipt', async () => {
diff --git a/control-ui/e2e/gfs-upload-v2.spec.ts b/control-ui/e2e/gfs-upload-v2.spec.ts
index ac665c15d..31f8c5dda 100644
--- a/control-ui/e2e/gfs-upload-v2.spec.ts
+++ b/control-ui/e2e/gfs-upload-v2.spec.ts
@@ -51,6 +51,44 @@ async function openWritableFolder(
return current
}
+/**
+ * Exercise the actual drop target while retaining Playwright's file-backed
+ * File object. The capture listener runs before the React change handler clears
+ * the native input, then the same file is delivered through a real DragEvent
+ * sequence to the visible drop zone. This keeps the journey user-facing while
+ * avoiding a synthetic upload request or direct storage mutation.
+ */
+async function dropFileIntoUploadDialog(
+ dialog: import('@playwright/test').Locator,
+ filePath: string
+): Promise<void> {
+ const input = dialog.getByLabel('Choose file to upload')
+ const initialDropzone = dialog.locator('label').filter({ hasText: 'Drag and drop' })
+ const labelFor = await initialDropzone.getAttribute('for')
+ if (!labelFor) throw new Error('upload drop zone has no associated file input')
+ await input.evaluate(element => {
+ element.addEventListener(
+ 'change',
+ event => {
+ const file = (event.currentTarget as HTMLInputElement).files?.[0]
+ if (file) (window as Window & { __gfsDropFile?: File }).__gfsDropFile = file
+ },
+ { capture: true, once: true }
+ )
+ })
+ await input.setInputFiles(filePath)
+ await dialog.locator(`label[for="${labelFor}"]`).evaluate(element => {
+ const file = (window as Window & { __gfsDropFile?: File }).__gfsDropFile
+ if (!file) throw new Error('file capture failed before drag-and-drop')
+ const dataTransfer = new DataTransfer()
+ dataTransfer.items.add(file)
+ for (const type of ['dragenter', 'dragover', 'drop']) {
+ element.dispatchEvent(new DragEvent(type, { bubbles: true, cancelable: true, dataTransfer }))
+ }
+ delete (window as Window & { __gfsDropFile?: File }).__gfsDropFile
+ })
+}
+
async function observeTwoProgressValues(
progress: import('@playwright/test').Locator,
timeout = 600_000
@@ -216,6 +254,64 @@ test.describe('GFS Upload v2 — Control UI large-upload project', () => {
await exerciseReplace(page, byteLength)
})
}
+
+ test('resumes the same persisted session after a visible drag-and-drop pause and reload', async ({
+ page,
+ }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-drag-resume')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ const source = await createDiskUploadFixture(32 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ await openWritableFolder(page, fixture.name)
+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()
+ let dialog = page.getByRole('dialog', { name: 'Upload file' })
+ await dropFileIntoUploadDialog(dialog, source.filePath)
+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()
+
+ const progress = page.getByRole('progressbar', {
+ name: `Upload progress for ${source.fileName}`,
+ })
+ await expect(progress).toBeVisible({ timeout: 60_000 })
+ await dialog.getByRole('button', { name: 'Pause', exact: true }).click()
+ await expect(dialog.getByRole('button', { name: 'Resume', exact: true })).toBeVisible({
+ timeout: 60_000,
+ })
+
+ await page.reload()
+ const resumedCurrent = await openWritableFolder(page, fixture.name)
+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()
+ dialog = page.getByRole('dialog', { name: 'Upload file' })
+ const statusResponse = page.waitForResponse(
+ response =>
+ response.request().method() === 'GET' &&
+ response.url().includes('/gfs/proxy/v1/uploads/') &&
+ response.url().includes('/status')
+ )
+ await dropFileIntoUploadDialog(dialog, source.filePath)
+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()
+ expect((await statusResponse).status()).toBe(200)
+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({
+ timeout: 600_000,
+ })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })
+ await expect(
+ resumedCurrent.getByRole('listitem').filter({ hasText: source.fileName })
+ ).toHaveCount(1)
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
})
test.describe('GFS Upload v2 — approved negative Control UI journeys', () => {
diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.ts
index 6e07dd637..26cf90781 100644
--- a/desktop-app/src/__tests__/appService.gfs.test.ts
+++ b/desktop-app/src/__tests__/appService.gfs.test.ts
@@ -411,16 +411,29 @@ describe('AppService GFS upload security scope', () => {
}
})
- it('maps a persisted active record to the renderer uploading state after restart', async () => {
+ it('maps a live persisted active record to the renderer uploading state', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-snapshot-state-'))
try {
const statePath = join(root, 'gfs-upload-sessions.json')
const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')
await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))
const service = authenticatedUploadService(statePath)
+ const job = {
+ snapshot: vi.fn(() => ({
+ state: 'uploading',
+ session: record.session,
+ uploadedBytes: record.session.committedBytes,
+ totalBytes: record.fileSize,
+ })),
+ }
+ service.gfsUploadJobs.set(record.uploadId, {
+ job,
+ promise: Promise.resolve(record.session),
+ scope: record.scope,
+ })
await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({
- state: 'suspended_auth',
+ state: 'uploading',
uploadedBytes: 0,
totalBytes: 4,
})
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index c43c7c19e..deae760e7 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.74",
+ "version": "0.1.75",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.74",
+ "version": "0.1.75",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index 5acf68823..2b5f2fb80 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.74",
+ "version": "0.1.75",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 2fb0956e3..76a00b2e0 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -8,7 +8,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.7.0',
- externalRestApiVersion: '0.1.74',
+ externalRestApiVersion: '0.1.75',
rpcProxyVersion: '0.1.64',
desktopVersion: '0.7.0',
minimumDesktopVersion: '0.1.252',
diff --git a/external-rest-api/test/routes.gfs.test.ts b/external-rest-api/test/routes.gfs.test.ts
index 6c5e3a14e..0973698b8 100644
--- a/external-rest-api/test/routes.gfs.test.ts
+++ b/external-rest-api/test/routes.gfs.test.ts
@@ -131,7 +131,7 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
expect(clientMock.controlApiStreamRequest).toHaveBeenCalledTimes(2)
})
- it('delegates missing and create-body-mismatched drive validation to control-api', async () => {
+ it('keeps capabilities drive-independent while control-api rejects mismatched creates', async () => {
clientMock.controlApiStreamRequest.mockImplementation(
async (
method: string,
@@ -140,13 +140,13 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
) => {
const drive = options.query?.drive
const body = typeof options.body === 'string' ? JSON.parse(options.body) : undefined
- if (!drive) {
- throw new ControlApiError('drive required', 400, { error: 'drive_required' })
- }
if (method === 'POST' && path === '/external/gfs/uploads' && body?.drive !== drive) {
throw new ControlApiError('drive mismatch', 400, { error: 'drive_mismatch' })
}
- return new Response(JSON.stringify({ ok: true }), { status: 200 })
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ })
}
)
const missing = await request(buildApp())
@@ -156,8 +156,8 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
.post('/me/gfs/uploads?drive=archive')
.set('authorization', 'Bearer sess-xyz')
.send({ drive: 'main', operation: 'create' })
- expect(missing.status).toBe(400)
- expect(missing.body).toEqual({ error: 'drive_required' })
+ expect(missing.status).toBe(200)
+ expect(missing.body).toEqual({ upload: { resumableV2: { enabled: true } } })
expect(mismatched.status).toBe(400)
expect(mismatched.body).toEqual({ error: 'drive_mismatch' })
expect(clientMock.controlApiStreamRequest).toHaveBeenCalledTimes(2)
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index 3319633d8..d92b8d7ce 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -56,7 +56,7 @@ class MemoryDb {
}
if (
sql.startsWith(
- "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()"
+ "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()"
)
) {
return { rows: [] }
@@ -369,6 +369,16 @@ class MemoryDb {
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
+ if (sql.includes("state = 'finalizing' AND session_epoch = $3")) {
+ if (row.state !== 'finalizing' || Number(row.session_epoch) !== Number(values[2])) {
+ return { rows: [] }
+ }
+ row.state = 'failed'
+ row.failure_code = values[1]
+ row.active_part_count = 0
+ row.finalizing_started_at = null
+ return { rows: [{ upload_id: row.upload_id }] }
+ }
row.state = 'failed'
row.failure_code = values[1]
row.active_part_count = 0
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index 6708af90f..a9d750ec3 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -1349,14 +1349,19 @@ export class GfsUploadSessionService {
await client.query(
`UPDATE gfs_upload_sessions
SET state = 'uploading', session_epoch = session_epoch + 1,
- finalizing_started_at = NULL, updated_at = now()
+ finalizing_started_at = NULL,
+ expires_at = GREATEST(
+ expires_at,
+ now() + ($2::bigint * interval '1 second')
+ ),
+ updated_at = now()
WHERE upload_id = $1 AND state = 'finalizing'`,
- [uploadId]
+ [uploadId, this.deps.config.sessionTtlSeconds]
)
}
const expired = await client.query(
`SELECT upload_id FROM gfs_upload_sessions
- WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()
+ WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()
LIMIT 100
FOR UPDATE SKIP LOCKED`,
[]
@@ -1628,16 +1633,19 @@ export class GfsUploadSessionService {
)
} catch (error) {
const failureCode = error instanceof GfsError ? error.code : 'finalization_failed'
+ let ownsFailureCleanup = false
await this.deps.tx.transaction(async client => {
const locked = await this.lockSession(client, uploadId, principal)
if (locked.state === 'finalizing' && locked.sessionEpoch === started.sessionEpoch) {
- await client.query(
+ const transitioned = await client.query(
`UPDATE gfs_upload_sessions
SET state = 'failed', failure_code = $2, active_part_count = 0,
finalizing_started_at = NULL, updated_at = now()
- WHERE upload_id = $1`,
- [uploadId, failureCode]
+ WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $3
+ RETURNING upload_id`,
+ [uploadId, failureCode, started.sessionEpoch]
)
+ ownsFailureCleanup = transitioned.rows.length === 1
}
})
// `deps.finalize` has already settled, so this controller no longer
@@ -1646,6 +1654,10 @@ export class GfsUploadSessionService {
// until the part timeout, preventing the direct failure cleanup below.
const current = this.inFlightFinalizers.get(uploadId)
if (current?.controller === finalizationAbort) this.inFlightFinalizers.delete(uploadId)
+ // A reclaimed epoch belongs to a newer attempt. The old finalizer may
+ // report its own failure, but it must never delete shared parts or the
+ // staging directory owned by the reclaimed session.
+ if (!ownsFailureCleanup) throw error
await this.waitForQuiescence(uploadId)
if (
!this.hasInFlight(uploadId) &&
@@ -1653,10 +1665,19 @@ export class GfsUploadSessionService {
) {
await this.deps.tx
.transaction(async client => {
- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
await client.query(
- `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`,
- [uploadId]
+ `DELETE FROM gfs_upload_parts
+ WHERE upload_id = $1
+ AND EXISTS (
+ SELECT 1 FROM gfs_upload_sessions
+ WHERE upload_id = $1 AND state = 'failed' AND session_epoch = $2
+ )`,
+ [uploadId, started.sessionEpoch]
+ )
+ await client.query(
+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now()
+ WHERE upload_id = $1 AND state = 'failed' AND session_epoch = $2`,
+ [uploadId, started.sessionEpoch]
)
})
.catch(() => undefined)