Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): harden resumable recovery contracts

Alfredo Lopez5f6035d30033
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
index 338d118bf..745795668 100644
--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
@@ -286,6 +286,41 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(status.session.activePartNumbers).toEqual([0, 1])
})
+ it('admits a part from durable reserved rows when the scalar counter is stale', async () => {
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: 'd'.repeat(32),
+ name: 'counter-drift-admission.bin',
+ sizeBytes: CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ await pool.query(
+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = 99 WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x44)
+ const result = await uploads.putPart(
+ created.session.uploadId,
+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ principal
+ )
+ expect(result.part.state).toBe('committed')
+ expect(result.session.activePartCount).toBe(0)
+ expect(
+ (
+ await pool.query(`SELECT active_part_count FROM gfs_upload_sessions WHERE upload_id = $1`, [
+ created.session.uploadId,
+ ])
+ ).rows[0]?.active_part_count
+ ).toBe(0)
+ })
+
it('serializes concurrent indexed part reservations without losing either commit', async () => {
const bytes = [
Buffer.alloc(CONFIG.preferredPartBytes, 0x31),
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index 99efa0e51..70ea22c0d 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -62,6 +62,7 @@ describe('isRetryableUploadStatus', () => {
class FakeXhr {
static requests: FakeXhr[] = []
static statuses: number[] = []
+ static statusesByPart: Record<number, number[]> = {}
static responseLosses = 0
upload: { onprogress: ((event: { lengthComputable: boolean; loaded: number }) => void) | null } =
{ onprogress: null }
@@ -72,8 +73,11 @@ class FakeXhr {
onabort: (() => void) | null = null
headers = new Map<string, string>()
body: Blob | null = null
+ url = ''
- open(): void {}
+ open(_method: string, url: string): void {
+ this.url = url
+ }
setRequestHeader(name: string, value: string): void {
this.headers.set(name, value)
}
@@ -82,7 +86,10 @@ class FakeXhr {
}
send(body: Blob): void {
this.body = body
- this.status = FakeXhr.statuses.shift() ?? 204
+ const partNumber = /\/parts\/(\d+)(?:\?|$)/.exec(this.url)?.[1]
+ const partStatuses =
+ partNumber === undefined ? undefined : FakeXhr.statusesByPart[Number(partNumber)]
+ this.status = partStatuses?.shift() ?? FakeXhr.statuses.shift() ?? 204
if (this.status === 429) this.headers.set('retry-after', '0')
FakeXhr.requests.push(this)
void body.arrayBuffer().then(() => {
@@ -108,6 +115,7 @@ describe('GfsUploadJob', () => {
beforeEach(() => {
FakeXhr.requests = []
FakeXhr.statuses = []
+ FakeXhr.statusesByPart = {}
FakeXhr.responseLosses = 0
vi.stubGlobal('XMLHttpRequest', FakeXhr)
})
@@ -770,6 +778,113 @@ describe('GfsUploadJob', () => {
expect(FakeXhr.requests).toHaveLength(3)
})
+ it('reconciles one ambiguous part against the live sibling lease set under four-way concurrency', async () => {
+ const file = new File([new Uint8Array([11, 12, 13, 14, 15, 16, 17, 18])], 'four-way.bin')
+ FakeXhr.statusesByPart = { 3: [500, 204] }
+ let statusCalls = 0
+ let completeCalls = 0
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(
+ JSON.stringify({
+ upload: {
+ resumableV2: {
+ enabled: true,
+ preferredChunkBytes: 2,
+ maxChunkBytes: 2,
+ maxConcurrentPartsPerSession: 4,
+ fallbackConcurrency: 2,
+ instabilityFailureThreshold: 3,
+ },
+ },
+ }),
+ { status: 200 }
+ )
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ uploadId: '35353535-3535-4353-8353-353535353535',
+ drive: 'main',
+ expectedBytes: 8,
+ partBytes: 2,
+ partCount: 4,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ },
+ }),
+ { status: 201 }
+ )
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status')) {
+ statusCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ session: {
+ uploadId: '35353535-3535-4353-8353-353535353535',
+ drive: 'main',
+ expectedBytes: 8,
+ partBytes: 2,
+ partCount: 4,
+ state: 'uploading',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 3,
+ activePartNumbers: [0, 1, 2],
+ },
+ parts: [],
+ },
+ }),
+ { status: 200 }
+ )
+ }
+ if (method === 'POST' && url.endsWith('/complete')) {
+ completeCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ uploadId: '35353535-3535-4353-8353-353535353535',
+ drive: 'main',
+ expectedBytes: 8,
+ partBytes: 2,
+ partCount: 4,
+ state: 'completed',
+ committedBytes: 8,
+ committedPartCount: 4,
+ activePartCount: 0,
+ },
+ }),
+ { status: 200 }
+ )
+ }
+ return new Response(null, { status: 204 })
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-four-way' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(statusCalls).toBe(1)
+ expect(completeCalls).toBe(1)
+ expect(FakeXhr.requests).toHaveLength(5)
+ })
+
it('persists an unresolved response-loss session without replaying its part', async () => {
const file = new File([new Uint8Array([9, 10])], 'outcome-unknown.bin', {
lastModified: 14,
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index c190ad8df..d149cfba2 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.72",
+ "version": "0.1.73",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.72",
+ "version": "0.1.73",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index 74887d709..2012fd655 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.72",
+ "version": "0.1.73",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.ts
index 48e652c45..2ceaeba2f 100644
--- a/external-rest-api/src/app.ts
+++ b/external-rest-api/src/app.ts
@@ -21,8 +21,11 @@ import { createExternalWorkflowsRouter } from './routes/workflows.js'
// Upload v2 part bodies are streamed octets and must not pass through the
// global JSON parser (which would buffer/reject the binary payload).
-const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
-const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/parts/[0-9]+$`, 'i')
+// Match the route shape, not only a valid UUID. Control API owns the UUID
+// validation and must receive malformed IDs as upload requests; otherwise a
+// binary body can be consumed by the global JSON parser and produce a parser
+// error before the canonical 4xx path is reached.
+const GFS_UPLOAD_PART_PATH = /^\/api\/v1\/me\/gfs\/uploads\/[^/]+\/parts\/[0-9]+$/i
export function createApp() {
const app = express()
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index ccc4ae588..4677676c6 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -8,7 +8,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.7.0',
- externalRestApiVersion: '0.1.72',
+ externalRestApiVersion: '0.1.73',
rpcProxyVersion: '0.1.64',
desktopVersion: '0.7.0',
minimumDesktopVersion: '0.1.252',
diff --git a/external-rest-api/test/app.test.ts b/external-rest-api/test/app.test.ts
index af93bbd21..34564057d 100644
--- a/external-rest-api/test/app.test.ts
+++ b/external-rest-api/test/app.test.ts
@@ -23,4 +23,15 @@ describe('external-rest-api app wiring', () => {
config.jsonBodyLimit = previousLimit
}
})
+
+ it('keeps malformed upload IDs on the binary route instead of the JSON parser', async () => {
+ const app = createApp()
+ await request(app)
+ .put('/api/v1/me/gfs/uploads/not-a-uuid/parts/0')
+ .set('content-type', 'application/json')
+ .send(Buffer.from([0xff, 0x00, 0x01]))
+ // Auth is intentionally the first route-level response. A 400 JSON
+ // parser error here would prove the binary bypass regressed.
+ .expect(401)
+ })
})
diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.ts
index 6c7e02a89..d46288c1d 100644
--- a/gfs-controller/src/config.test.ts
+++ b/gfs-controller/src/config.test.ts
@@ -170,4 +170,16 @@ describe('GFS_UPLOAD_V2 strict disabled contract', () => {
)
}
})
+
+ it('fails closed when stale lease cleanup could race an active part timeout', () => {
+ vi.stubEnv('GFS_DEV_MODE', 'true')
+ vi.stubEnv('GFS_UPLOAD_PART_TIMEOUT_MS', '600000')
+ vi.stubEnv('GFS_UPLOAD_STALE_PART_LEASE_MS', '600000')
+ expect(() => loadConfig()).toThrow(
+ /GFS_UPLOAD_STALE_PART_LEASE_MS must be greater than GFS_UPLOAD_PART_TIMEOUT_MS/
+ )
+
+ vi.stubEnv('GFS_UPLOAD_STALE_PART_LEASE_MS', '600001')
+ expect(loadConfig().uploadV2.stalePartLeaseMs).toBe(600001)
+ })
})
diff --git a/gfs-controller/src/config.ts b/gfs-controller/src/config.ts
index eb54d4ab7..76da00817 100644
--- a/gfs-controller/src/config.ts
+++ b/gfs-controller/src/config.ts
@@ -320,6 +320,11 @@ function uploadConfig(): GfsUploadConfig {
const partTimeoutMs = uploadDurationMs('GFS_UPLOAD_PART_TIMEOUT_MS', GFS_UPLOAD_V2_PART_TIMEOUT_MS, 24 * 60 * 60 * 1000)
const finalizeTimeoutMs = uploadDurationMs('GFS_UPLOAD_FINALIZE_TIMEOUT_MS', GFS_UPLOAD_V2_FINALIZE_TIMEOUT_MS, 24 * 60 * 60 * 1000)
const stalePartLeaseMs = uploadDurationMs('GFS_UPLOAD_STALE_PART_LEASE_MS', GFS_UPLOAD_V2_STALE_PART_LEASE_MS, 24 * 60 * 60 * 1000)
+ if (stalePartLeaseMs <= partTimeoutMs) {
+ throw new Error(
+ '[gfsc] GFS_UPLOAD_STALE_PART_LEASE_MS must be greater than GFS_UPLOAD_PART_TIMEOUT_MS'
+ )
+ }
const receiptRetentionMs = uploadDurationMsWithAlias(
'GFS_UPLOAD_COMPLETED_RECEIPT_TTL_MS',
'GFS_UPLOAD_RECEIPT_RETENTION_SECONDS',
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index 263d8f253..ab83a8ef1 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -123,6 +123,16 @@ class MemoryDb {
return { rows: [{ count: String(active) }] }
}
if (sql.startsWith('SELECT COUNT(*)::bigint')) {
+ if (sql.includes('FROM gfs_upload_parts WHERE upload_id = $1 AND state = \'reserved\'')) {
+ const uploadId = String(values[0])
+ return {
+ rows: [{
+ count: String(
+ this.parts.filter(part => part.upload_id === uploadId && part.state === 'reserved').length
+ ),
+ }],
+ }
+ }
const drive = String(values[0])
return {
rows: [
@@ -265,7 +275,9 @@ class MemoryDb {
row.finalizing_started_at = null
return { rows: [] }
}
- row.active_part_count = Number(row.active_part_count) + 1
+ row.active_part_count = sql.includes('active_part_count = (')
+ ? this.parts.filter(part => part.upload_id === row.upload_id && part.state === 'reserved').length
+ : Number(row.active_part_count) + 1
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'committed'")) {
@@ -338,6 +350,21 @@ class MemoryDb {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
row.state = 'finalizing'
row.finalizing_started_at = new Date().toISOString()
+ const extensionMs = Number(values[2] ?? 0)
+ const currentExpiry = Date.parse(String(row.expires_at))
+ const finalizerExpiry = Date.now() + extensionMs
+ row.expires_at = new Date(Math.max(currentExpiry, finalizerExpiry)).toISOString()
+ return { rows: [] }
+ }
+ if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'completed'")) {
+ const [uploadId, resourceId, version, sha256] = values
+ const row = this.sessions.find(session => session.upload_id === String(uploadId))!
+ row.state = 'completed'
+ row.result_resource_id = resourceId
+ row.result_version = version
+ row.result_sha256 = sha256
+ row.completed_at = new Date().toISOString()
+ row.finalizing_started_at = null
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {
@@ -588,6 +615,69 @@ describe('GfsUploadSessionService', () => {
expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')
})
+ it('admits a new part from durable reserved rows when the session counter is stale', async () => {
+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-counter-drift-'))
+ const db = new MemoryDb()
+ const config = { ...TEST_CONFIG, maxConcurrentPartsPerSession: 1 }
+ const uploads = service(db, config)
+ const created = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'counter-drift.bin',
+ sizeBytes: TEST_CONFIG.preferredPartBytes,
+ idempotencyKey: '68686868-6868-4686-8686-686868686868',
+ })
+
+ // A stale scalar must not deny admission when no durable reserved row
+ // exists. This is the inverse of the double-decrement regression: rows,
+ // not a receipt field, own the concurrency contract.
+ db.sessions[0]!.active_part_count = 99
+ const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x52)
+ const committed = await uploads.putPart(
+ created.session.uploadId,
+ partGeometry(
+ { expectedBytes: TEST_CONFIG.preferredPartBytes, partBytes: TEST_CONFIG.preferredPartBytes },
+ 0
+ ),
+ createSha(payload),
+ Readable.from([payload]) as never,
+ principal
+ )
+
+ expect(committed.part.state).toBe('committed')
+ expect(committed.session.activePartCount).toBe(0)
+ expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })
+ })
+
+ it('extends the durable expiry window before entering finalization', async () => {
+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-ttl-'))
+ const db = new MemoryDb()
+ const config = { ...TEST_CONFIG, sessionTtlSeconds: 1, finalizeTimeoutMs: 600_000 }
+ let finalizerExpiry: number | undefined
+ const uploads = service(db, config, {
+ finalize: async session => {
+ finalizerExpiry = Date.parse(session.expiresAt)
+ return { resourceId: 'resource-finalizing-ttl', version: 1, sha256: 'a'.repeat(64) }
+ },
+ })
+ const created = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'finalizing-ttl.bin',
+ sizeBytes: 0,
+ idempotencyKey: '69696969-6969-4696-8696-696969696969',
+ })
+ const originalExpiry = Date.parse(created.session.expiresAt)
+
+ await expect(uploads.complete(created.session.uploadId, principal)).resolves.toMatchObject({
+ state: 'completed',
+ })
+ expect(finalizerExpiry).toBeGreaterThan(originalExpiry)
+ expect(Date.parse(db.sessions[0]!.expires_at as string)).toBeGreaterThan(originalExpiry)
+ })
+
it('removes a rename-before-commit orphan while the reserved lease is still locked', async () => {
tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-orphan-'))
const db = new MemoryDb()
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index 409c07c69..6708af90f 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -843,7 +843,17 @@ export class GfsUploadSessionService {
'active_part_streams_global'
)
}
- if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession)
+ // The scalar on the session is a receipt/cache field. Admission must
+ // use the locked part rows as the authority so an outcome-unknown
+ // commit or a stale response can never resurrect a false slot.
+ const activeSession = await client.query(
+ `SELECT COUNT(*)::bigint AS count
+ FROM gfs_upload_parts
+ WHERE upload_id = $1 AND state = 'reserved'`,
+ [uploadId]
+ )
+ const activeSessionCount = Number(activeSession.rows[0]?.count ?? 0)
+ if (activeSessionCount >= this.deps.config.maxConcurrentPartsPerSession)
throw quotaExceeded(
'per-session part concurrency limit reached',
'active_part_streams_session'
@@ -897,8 +907,9 @@ export class GfsUploadSessionService {
WHERE upload_id = $1`,
[uploadId]
)
+ const refreshed = await this.lockSession(client, uploadId, principal)
return {
- session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 },
+ session: refreshed,
existing: null as UploadPartRow | null,
stagingPath,
leaseEpoch,
@@ -1564,11 +1575,23 @@ export class GfsUploadSessionService {
)
}
await client.query(
- `UPDATE gfs_upload_sessions SET state = 'finalizing', finalizing_started_at = now(), updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,
- [uploadId, locked.sessionEpoch]
+ `UPDATE gfs_upload_sessions
+ SET state = 'finalizing',
+ finalizing_started_at = now(),
+ expires_at = GREATEST(
+ expires_at,
+ now() + ($3::bigint * interval '1 millisecond')
+ ),
+ updated_at = now()
+ WHERE upload_id = $1 AND session_epoch = $2`,
+ [uploadId, locked.sessionEpoch, this.deps.config.finalizeTimeoutMs]
)
+ // Re-read the row while the transaction lock is held. The finalizer
+ // and any receipt now carry the durable deadline, not the pre-
+ // finalization session TTL that could expire during publication.
+ const refreshed = await this.lockSession(client, uploadId, principal)
return {
- ...locked,
+ ...refreshed,
wholeSha256: expectedWhole ?? locked.wholeSha256,
state: 'finalizing' as const,
parts: committedParts,