Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): harden resumable recovery contracts
Alfredo Lopez5f6035d30033
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.tsindex 338d118bf..745795668 100644--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts@@ -286,6 +286,41 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {expect(status.session.activePartNumbers).toEqual([0, 1])})+ it('admits a part from durable reserved rows when the scalar counter is stale', async () => {+ const created = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: 'd'.repeat(32),+ name: 'counter-drift-admission.bin',+ sizeBytes: CONFIG.preferredPartBytes,+ idempotencyKey: randomUUID(),+ })+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ await pool.query(+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = 99 WHERE upload_id = $1`,+ [created.session.uploadId]+ )+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x44)+ const result = await uploads.putPart(+ created.session.uploadId,+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),+ checksum(bytes),+ Readable.from([bytes]) as never,+ principal+ )+ expect(result.part.state).toBe('committed')+ expect(result.session.activePartCount).toBe(0)+ expect(+ (+ await pool.query(`SELECT active_part_count FROM gfs_upload_sessions WHERE upload_id = $1`, [+ created.session.uploadId,+ ])+ ).rows[0]?.active_part_count+ ).toBe(0)+ })+it('serializes concurrent indexed part reservations without losing either commit', async () => {const bytes = [Buffer.alloc(CONFIG.preferredPartBytes, 0x31),diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.tsindex 99efa0e51..70ea22c0d 100644--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts@@ -62,6 +62,7 @@ describe('isRetryableUploadStatus', () => {class FakeXhr {static requests: FakeXhr[] = []static statuses: number[] = []+ static statusesByPart: Record<number, number[]> = {}static responseLosses = 0upload: { onprogress: ((event: { lengthComputable: boolean; loaded: number }) => void) | null } ={ onprogress: null }@@ -72,8 +73,11 @@ class FakeXhr {onabort: (() => void) | null = nullheaders = new Map<string, string>()body: Blob | null = null+ url = ''- open(): void {}+ open(_method: string, url: string): void {+ this.url = url+ }setRequestHeader(name: string, value: string): void {this.headers.set(name, value)}@@ -82,7 +86,10 @@ class FakeXhr {}send(body: Blob): void {this.body = body- this.status = FakeXhr.statuses.shift() ?? 204+ const partNumber = /\/parts\/(\d+)(?:\?|$)/.exec(this.url)?.[1]+ const partStatuses =+ partNumber === undefined ? undefined : FakeXhr.statusesByPart[Number(partNumber)]+ this.status = partStatuses?.shift() ?? FakeXhr.statuses.shift() ?? 204if (this.status === 429) this.headers.set('retry-after', '0')FakeXhr.requests.push(this)void body.arrayBuffer().then(() => {@@ -108,6 +115,7 @@ describe('GfsUploadJob', () => {beforeEach(() => {FakeXhr.requests = []FakeXhr.statuses = []+ FakeXhr.statusesByPart = {}FakeXhr.responseLosses = 0vi.stubGlobal('XMLHttpRequest', FakeXhr)})@@ -770,6 +778,113 @@ describe('GfsUploadJob', () => {expect(FakeXhr.requests).toHaveLength(3)})+ it('reconciles one ambiguous part against the live sibling lease set under four-way concurrency', async () => {+ const file = new File([new Uint8Array([11, 12, 13, 14, 15, 16, 17, 18])], 'four-way.bin')+ FakeXhr.statusesByPart = { 3: [500, 204] }+ let statusCalls = 0+ let completeCalls = 0+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(+ JSON.stringify({+ upload: {+ resumableV2: {+ enabled: true,+ preferredChunkBytes: 2,+ maxChunkBytes: 2,+ maxConcurrentPartsPerSession: 4,+ fallbackConcurrency: 2,+ instabilityFailureThreshold: 3,+ },+ },+ }),+ { status: 200 }+ )+ if (method === 'POST' && url.endsWith('/uploads'))+ return new Response(+ JSON.stringify({+ ok: true,+ data: {+ uploadId: '35353535-3535-4353-8353-353535353535',+ drive: 'main',+ expectedBytes: 8,+ partBytes: 2,+ partCount: 4,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ },+ }),+ { status: 201 }+ )+ if (method === 'HEAD') return new Response(null, { status: 204 })+ if (method === 'GET' && url.includes('/status')) {+ statusCalls += 1+ return new Response(+ JSON.stringify({+ ok: true,+ data: {+ session: {+ uploadId: '35353535-3535-4353-8353-353535353535',+ drive: 'main',+ expectedBytes: 8,+ partBytes: 2,+ partCount: 4,+ state: 'uploading',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 3,+ activePartNumbers: [0, 1, 2],+ },+ parts: [],+ },+ }),+ { status: 200 }+ )+ }+ if (method === 'POST' && url.endsWith('/complete')) {+ completeCalls += 1+ return new Response(+ JSON.stringify({+ ok: true,+ data: {+ uploadId: '35353535-3535-4353-8353-353535353535',+ drive: 'main',+ expectedBytes: 8,+ partBytes: 2,+ partCount: 4,+ state: 'completed',+ committedBytes: 8,+ committedPartCount: 4,+ activePartCount: 0,+ },+ }),+ { status: 200 }+ )+ }+ return new Response(null, { status: 204 })+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-four-way' },+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(statusCalls).toBe(1)+ expect(completeCalls).toBe(1)+ expect(FakeXhr.requests).toHaveLength(5)+ })+it('persists an unresolved response-loss session without replaying its part', async () => {const file = new File([new Uint8Array([9, 10])], 'outcome-unknown.bin', {lastModified: 14,diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex c190ad8df..d149cfba2 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.72",+ "version": "0.1.73","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.72",+ "version": "0.1.73","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex 74887d709..2012fd655 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.72",+ "version": "0.1.73","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.tsindex 48e652c45..2ceaeba2f 100644--- a/external-rest-api/src/app.ts+++ b/external-rest-api/src/app.ts@@ -21,8 +21,11 @@ import { createExternalWorkflowsRouter } from './routes/workflows.js'// Upload v2 part bodies are streamed octets and must not pass through the// global JSON parser (which would buffer/reject the binary payload).-const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'-const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/parts/[0-9]+$`, 'i')+// Match the route shape, not only a valid UUID. Control API owns the UUID+// validation and must receive malformed IDs as upload requests; otherwise a+// binary body can be consumed by the global JSON parser and produce a parser+// error before the canonical 4xx path is reached.+const GFS_UPLOAD_PART_PATH = /^\/api\/v1\/me\/gfs\/uploads\/[^/]+\/parts\/[0-9]+$/iexport function createApp() {const app = express()diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex ccc4ae588..4677676c6 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -8,7 +8,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.7.0',- externalRestApiVersion: '0.1.72',+ externalRestApiVersion: '0.1.73',rpcProxyVersion: '0.1.64',desktopVersion: '0.7.0',minimumDesktopVersion: '0.1.252',diff --git a/external-rest-api/test/app.test.ts b/external-rest-api/test/app.test.tsindex af93bbd21..34564057d 100644--- a/external-rest-api/test/app.test.ts+++ b/external-rest-api/test/app.test.ts@@ -23,4 +23,15 @@ describe('external-rest-api app wiring', () => {config.jsonBodyLimit = previousLimit}})++ it('keeps malformed upload IDs on the binary route instead of the JSON parser', async () => {+ const app = createApp()+ await request(app)+ .put('/api/v1/me/gfs/uploads/not-a-uuid/parts/0')+ .set('content-type', 'application/json')+ .send(Buffer.from([0xff, 0x00, 0x01]))+ // Auth is intentionally the first route-level response. A 400 JSON+ // parser error here would prove the binary bypass regressed.+ .expect(401)+ })})diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.tsindex 6c7e02a89..d46288c1d 100644--- a/gfs-controller/src/config.test.ts+++ b/gfs-controller/src/config.test.ts@@ -170,4 +170,16 @@ describe('GFS_UPLOAD_V2 strict disabled contract', () => {)}})++ it('fails closed when stale lease cleanup could race an active part timeout', () => {+ vi.stubEnv('GFS_DEV_MODE', 'true')+ vi.stubEnv('GFS_UPLOAD_PART_TIMEOUT_MS', '600000')+ vi.stubEnv('GFS_UPLOAD_STALE_PART_LEASE_MS', '600000')+ expect(() => loadConfig()).toThrow(+ /GFS_UPLOAD_STALE_PART_LEASE_MS must be greater than GFS_UPLOAD_PART_TIMEOUT_MS/+ )++ vi.stubEnv('GFS_UPLOAD_STALE_PART_LEASE_MS', '600001')+ expect(loadConfig().uploadV2.stalePartLeaseMs).toBe(600001)+ })})diff --git a/gfs-controller/src/config.ts b/gfs-controller/src/config.tsindex eb54d4ab7..76da00817 100644--- a/gfs-controller/src/config.ts+++ b/gfs-controller/src/config.ts@@ -320,6 +320,11 @@ function uploadConfig(): GfsUploadConfig {const partTimeoutMs = uploadDurationMs('GFS_UPLOAD_PART_TIMEOUT_MS', GFS_UPLOAD_V2_PART_TIMEOUT_MS, 24 * 60 * 60 * 1000)const finalizeTimeoutMs = uploadDurationMs('GFS_UPLOAD_FINALIZE_TIMEOUT_MS', GFS_UPLOAD_V2_FINALIZE_TIMEOUT_MS, 24 * 60 * 60 * 1000)const stalePartLeaseMs = uploadDurationMs('GFS_UPLOAD_STALE_PART_LEASE_MS', GFS_UPLOAD_V2_STALE_PART_LEASE_MS, 24 * 60 * 60 * 1000)+ if (stalePartLeaseMs <= partTimeoutMs) {+ throw new Error(+ '[gfsc] GFS_UPLOAD_STALE_PART_LEASE_MS must be greater than GFS_UPLOAD_PART_TIMEOUT_MS'+ )+ }const receiptRetentionMs = uploadDurationMsWithAlias('GFS_UPLOAD_COMPLETED_RECEIPT_TTL_MS','GFS_UPLOAD_RECEIPT_RETENTION_SECONDS',diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex 263d8f253..ab83a8ef1 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -123,6 +123,16 @@ class MemoryDb {return { rows: [{ count: String(active) }] }}if (sql.startsWith('SELECT COUNT(*)::bigint')) {+ if (sql.includes('FROM gfs_upload_parts WHERE upload_id = $1 AND state = \'reserved\'')) {+ const uploadId = String(values[0])+ return {+ rows: [{+ count: String(+ this.parts.filter(part => part.upload_id === uploadId && part.state === 'reserved').length+ ),+ }],+ }+ }const drive = String(values[0])return {rows: [@@ -265,7 +275,9 @@ class MemoryDb {row.finalizing_started_at = nullreturn { rows: [] }}- row.active_part_count = Number(row.active_part_count) + 1+ row.active_part_count = sql.includes('active_part_count = (')+ ? this.parts.filter(part => part.upload_id === row.upload_id && part.state === 'reserved').length+ : Number(row.active_part_count) + 1return { rows: [] }}if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'committed'")) {@@ -338,6 +350,21 @@ class MemoryDb {const row = this.sessions.find(session => session.upload_id === String(values[0]))!row.state = 'finalizing'row.finalizing_started_at = new Date().toISOString()+ const extensionMs = Number(values[2] ?? 0)+ const currentExpiry = Date.parse(String(row.expires_at))+ const finalizerExpiry = Date.now() + extensionMs+ row.expires_at = new Date(Math.max(currentExpiry, finalizerExpiry)).toISOString()+ return { rows: [] }+ }+ if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'completed'")) {+ const [uploadId, resourceId, version, sha256] = values+ const row = this.sessions.find(session => session.upload_id === String(uploadId))!+ row.state = 'completed'+ row.result_resource_id = resourceId+ row.result_version = version+ row.result_sha256 = sha256+ row.completed_at = new Date().toISOString()+ row.finalizing_started_at = nullreturn { rows: [] }}if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {@@ -588,6 +615,69 @@ describe('GfsUploadSessionService', () => {expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')})+ it('admits a new part from durable reserved rows when the session counter is stale', async () => {+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-counter-drift-'))+ const db = new MemoryDb()+ const config = { ...TEST_CONFIG, maxConcurrentPartsPerSession: 1 }+ const uploads = service(db, config)+ const created = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'counter-drift.bin',+ sizeBytes: TEST_CONFIG.preferredPartBytes,+ idempotencyKey: '68686868-6868-4686-8686-686868686868',+ })++ // A stale scalar must not deny admission when no durable reserved row+ // exists. This is the inverse of the double-decrement regression: rows,+ // not a receipt field, own the concurrency contract.+ db.sessions[0]!.active_part_count = 99+ const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x52)+ const committed = await uploads.putPart(+ created.session.uploadId,+ partGeometry(+ { expectedBytes: TEST_CONFIG.preferredPartBytes, partBytes: TEST_CONFIG.preferredPartBytes },+ 0+ ),+ createSha(payload),+ Readable.from([payload]) as never,+ principal+ )++ expect(committed.part.state).toBe('committed')+ expect(committed.session.activePartCount).toBe(0)+ expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })+ })++ it('extends the durable expiry window before entering finalization', async () => {+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-ttl-'))+ const db = new MemoryDb()+ const config = { ...TEST_CONFIG, sessionTtlSeconds: 1, finalizeTimeoutMs: 600_000 }+ let finalizerExpiry: number | undefined+ const uploads = service(db, config, {+ finalize: async session => {+ finalizerExpiry = Date.parse(session.expiresAt)+ return { resourceId: 'resource-finalizing-ttl', version: 1, sha256: 'a'.repeat(64) }+ },+ })+ const created = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'finalizing-ttl.bin',+ sizeBytes: 0,+ idempotencyKey: '69696969-6969-4696-8696-696969696969',+ })+ const originalExpiry = Date.parse(created.session.expiresAt)++ await expect(uploads.complete(created.session.uploadId, principal)).resolves.toMatchObject({+ state: 'completed',+ })+ expect(finalizerExpiry).toBeGreaterThan(originalExpiry)+ expect(Date.parse(db.sessions[0]!.expires_at as string)).toBeGreaterThan(originalExpiry)+ })+it('removes a rename-before-commit orphan while the reserved lease is still locked', async () => {tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-orphan-'))const db = new MemoryDb()diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex 409c07c69..6708af90f 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -843,7 +843,17 @@ export class GfsUploadSessionService {'active_part_streams_global')}- if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession)+ // The scalar on the session is a receipt/cache field. Admission must+ // use the locked part rows as the authority so an outcome-unknown+ // commit or a stale response can never resurrect a false slot.+ const activeSession = await client.query(+ `SELECT COUNT(*)::bigint AS count+ FROM gfs_upload_parts+ WHERE upload_id = $1 AND state = 'reserved'`,+ [uploadId]+ )+ const activeSessionCount = Number(activeSession.rows[0]?.count ?? 0)+ if (activeSessionCount >= this.deps.config.maxConcurrentPartsPerSession)throw quotaExceeded('per-session part concurrency limit reached','active_part_streams_session'@@ -897,8 +907,9 @@ export class GfsUploadSessionService {WHERE upload_id = $1`,[uploadId])+ const refreshed = await this.lockSession(client, uploadId, principal)return {- session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 },+ session: refreshed,existing: null as UploadPartRow | null,stagingPath,leaseEpoch,@@ -1564,11 +1575,23 @@ export class GfsUploadSessionService {)}await client.query(- `UPDATE gfs_upload_sessions SET state = 'finalizing', finalizing_started_at = now(), updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,- [uploadId, locked.sessionEpoch]+ `UPDATE gfs_upload_sessions+ SET state = 'finalizing',+ finalizing_started_at = now(),+ expires_at = GREATEST(+ expires_at,+ now() + ($3::bigint * interval '1 millisecond')+ ),+ updated_at = now()+ WHERE upload_id = $1 AND session_epoch = $2`,+ [uploadId, locked.sessionEpoch, this.deps.config.finalizeTimeoutMs])+ // Re-read the row while the transaction lock is held. The finalizer+ // and any receipt now carry the durable deadline, not the pre-+ // finalization session TTL that could expire during publication.+ const refreshed = await this.lockSession(client, uploadId, principal)return {- ...locked,+ ...refreshed,wholeSha256: expectedWhole ?? locked.wholeSha256,state: 'finalizing' as const,parts: committedParts,