Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): close resumable recovery contracts

Alfredo Lopez0f9d183aa7b1
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 1f097611d..2bf7dfede 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -7,6 +7,7 @@ import {
import { config } from './config.js'
import {
applyGfsUploadCleanupSchema,
+ applyGfsUploadFinalizingSchema,
applyGfsUploadSessionSchema,
} from './services/gfsUploadSchema.js'
import { applyMemberRegistrationCredentialsSchema } from './services/memberRegistrationCredentialsSchema.js'
@@ -5475,6 +5476,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
version: '0092_gfs_upload_cleanup_receipt',
apply: applyGfsUploadCleanupSchema,
},
+ {
+ version: '0093_gfs_upload_finalizing_recovery',
+ apply: applyGfsUploadFinalizingSchema,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.ts
index 8a9e4aea4..c03065872 100644
--- a/control-api/src/routes/external/gfs.ts
+++ b/control-api/src/routes/external/gfs.ts
@@ -279,10 +279,24 @@ export function createExternalGfsRouter(): Router {
// writer-scoped token and never fall back to the legacy JSON/base64 path.
router.get(
'/external/gfs/capabilities',
- requireCanonicalUploadDrive,
gfsUploadAdmission,
asyncHandler(async (req: ExternalGfsUploadRequest, res) => {
- await proxyUploadToGfsc(req, res, canonicalUploadDrive(req), 'GET', '/v1/capabilities')
+ // Capabilities describe the protocol, not a drive-scoped session. Keep
+ // an optional drive for token routing when supplied, but a global probe
+ // must remain valid without query parameters.
+ const rawQueryDrive = req.query.drive
+ if (rawQueryDrive !== undefined) {
+ if (
+ typeof rawQueryDrive !== 'string' ||
+ rawQueryDrive.length === 0 ||
+ rawQueryDrive !== rawQueryDrive.trim()
+ ) {
+ res.status(400).json({ error: 'drive_invalid' })
+ return
+ }
+ }
+ const drive = rawQueryDrive === undefined ? GFS_DEFAULT_DRIVE : rawQueryDrive
+ await proxyUploadToGfsc(req, res, drive, 'GET', '/v1/capabilities')
})
)
router.post(
diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.ts
index 45e2f00ec..4ce938444 100644
--- a/control-api/src/routes/gfs/proxy.ts
+++ b/control-api/src/routes/gfs/proxy.ts
@@ -28,6 +28,8 @@ const PASSTHROUGH_RESPONSE_HEADERS = [
'upload-part-length',
'upload-checksum',
'retry-after',
+ 'x-ratelimit-limit',
+ 'x-ratelimit-remaining',
]
const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
const UPLOAD_PART_PATH = new RegExp(`^/v1/uploads/${UUID_PATH}/parts/[0-9]+(?:\\?|$)`, 'i')
diff --git a/control-api/src/services/gfsUploadSchema.ts b/control-api/src/services/gfsUploadSchema.ts
index aec134997..4d9e4f14f 100644
--- a/control-api/src/services/gfsUploadSchema.ts
+++ b/control-api/src/services/gfsUploadSchema.ts
@@ -44,6 +44,7 @@ export async function applyGfsUploadSessionSchema(db: DbClient): Promise<void> {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ NULL,
+ finalizing_started_at TIMESTAMPTZ NULL,
cleanup_at TIMESTAMPTZ NULL,
CONSTRAINT gfs_upload_sessions_owner_idempotency_unique
UNIQUE (owner_subject, drive, idempotency_key)
@@ -87,3 +88,10 @@ export async function applyGfsUploadCleanupSchema(db: DbClient): Promise<void> {
`ALTER TABLE IF EXISTS gfs_upload_sessions ADD COLUMN IF NOT EXISTS cleanup_at TIMESTAMPTZ NULL`
)
}
+
+/** Additive follow-up for crash recovery of a session fenced in finalization. */
+export async function applyGfsUploadFinalizingSchema(db: DbClient): Promise<void> {
+ await db.query(
+ `ALTER TABLE IF EXISTS gfs_upload_sessions ADD COLUMN IF NOT EXISTS finalizing_started_at TIMESTAMPTZ NULL`
+ )
+}
diff --git a/control-api/test/db.gfsUploadSessionMigration.test.ts b/control-api/test/db.gfsUploadSessionMigration.test.ts
index 13eefce51..5c15fd7d0 100644
--- a/control-api/test/db.gfsUploadSessionMigration.test.ts
+++ b/control-api/test/db.gfsUploadSessionMigration.test.ts
@@ -61,4 +61,14 @@ describe('0091_gfs_upload_sessions', () => {
['gfs_upload_sessions', 'none'],
])
})
+
+ it('registers the additive finalizing recovery migration', () => {
+ const source = readFileSync(new URL('../src/db.ts', import.meta.url), 'utf8')
+ const schema = readFileSync(
+ new URL('../src/services/gfsUploadSchema.ts', import.meta.url),
+ 'utf8'
+ )
+ expect(source).toContain("version: '0093_gfs_upload_finalizing_recovery'")
+ expect(schema).toContain('finalizing_started_at TIMESTAMPTZ')
+ })
})
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
index 06771f2f9..0a52804e4 100644
--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
@@ -248,6 +248,44 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(rows.rows).toEqual(expect.arrayContaining([{ state: 'committed', count: 2 }]))
})
+ it('reports the exact reserved part set while sibling streams are active', async () => {
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: 'c'.repeat(32),
+ name: 'active-part-set.bin',
+ sizeBytes: 2 * CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ await pool.query(
+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = 2 WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ await pool.query(
+ `INSERT INTO gfs_upload_parts
+ (upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch, lease_started_at)
+ VALUES
+ ($1, 0, 0, $2, $3, 'reserved', $4, 1, now()),
+ ($1, 1, $2, $2, $3, 'reserved', $5, 1, now())`,
+ [
+ created.session.uploadId,
+ CONFIG.preferredPartBytes,
+ 'a'.repeat(64),
+ join(tempRoot, '.uploads', created.session.uploadId, 'parts', '0.part.tmp-test'),
+ join(tempRoot, '.uploads', created.session.uploadId, 'parts', '1.part.tmp-test'),
+ ]
+ )
+ const status = await uploads.status(created.session.uploadId, {
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ })
+ expect(status.session.activePartCount).toBe(2)
+ expect(status.session.activePartNumbers).toEqual([0, 1])
+ })
+
it('serializes concurrent indexed part reservations without losing either commit', async () => {
const bytes = [
Buffer.alloc(CONFIG.preferredPartBytes, 0x31),
@@ -641,6 +679,39 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(corruptRow.rows[0]).toEqual({ state: 'failed', failure_code: 'corrupt_part_missing' })
})
+ it('reopens an orphaned finalizing session after the durable finalizer lease expires', async () => {
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: 'a'.repeat(32),
+ name: 'orphan-finalizer.bin',
+ sizeBytes: 0,
+ idempotencyKey: randomUUID(),
+ })
+ const before = await pool.query(
+ `SELECT session_epoch FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ await pool.query(
+ `UPDATE gfs_upload_sessions
+ SET state = 'finalizing', finalizing_started_at = now() - interval '1 hour'
+ WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ await uploads.reconcile()
+ const row = await pool.query(
+ `SELECT state, session_epoch, finalizing_started_at FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ expect(row.rows[0]).toEqual({
+ state: 'uploading',
+ session_epoch: Number(before.rows[0]?.session_epoch) + 1,
+ finalizing_started_at: null,
+ })
+ })
+
it('rejects cancel during finalization and leaves one completed receipt', async () => {
let finalizerStarted!: () => void
let releaseFinalizer!: () => void
diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.ts
index e7bf7c3b4..f29fc5f4b 100644
--- a/control-api/test/routes.externalGfs.test.ts
+++ b/control-api/test/routes.externalGfs.test.ts
@@ -265,9 +265,15 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)
})
describe('indexed upload relay canonical drive', () => {
- it('rejects a missing or create-body-mismatched drive before minting a writer token', async () => {
+ it('allows a drive-independent capability probe while still rejecting mismatched creates', async () => {
auth()
- const fetchMock = vi.fn()
+ const fetchMock = vi.fn(
+ async () =>
+ new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ })
+ )
vi.stubGlobal('fetch', fetchMock)
const app = await buildApp()
@@ -279,12 +285,16 @@ describe('indexed upload relay canonical drive', () => {
.set('x-user-session-token', 'sess')
.send({ drive: 'main', operation: 'create' })
- expect(missing.status).toBe(400)
- expect(missing.body).toEqual({ error: 'drive_required' })
+ expect(missing.status).toBe(200)
+ expect(missing.body).toMatchObject({ upload: { resumableV2: { enabled: true } } })
expect(mismatched.status).toBe(400)
expect(mismatched.body).toEqual({ error: 'drive_mismatch' })
- expect(mockSignGfsToken).not.toHaveBeenCalled()
- expect(fetchMock).not.toHaveBeenCalled()
+ expect(mockSignGfsToken).toHaveBeenCalledWith({
+ subject: U1,
+ drive: 'main',
+ scopes: ['gfs.write'],
+ })
+ expect(fetchMock).toHaveBeenCalledTimes(1)
})
it('rejects malformed canonical drive values before admission or token minting', async () => {
@@ -299,16 +309,14 @@ describe('indexed upload relay canonical drive', () => {
.get(`/external/gfs/capabilities?drive=${encodeURIComponent(drive)}`)
.set('x-user-session-token', 'sess')
expect(response.status).toBe(400)
- expect(response.body).toEqual({ error: 'drive_required' })
+ expect(response.body).toEqual({ error: 'drive_invalid' })
}
const arrayValue = await request(await buildApp())
.get('/external/gfs/capabilities?drive=archive&drive=main')
.set('x-user-session-token', 'sess')
expect(arrayValue.status).toBe(400)
- expect(arrayValue.body).toEqual({ error: 'drive_required' })
- expect(mockSignGfsToken).not.toHaveBeenCalled()
- expect(fetchMock).not.toHaveBeenCalled()
+ expect(arrayValue.body).toEqual({ error: 'drive_invalid' })
})
it('signs and forwards every lifecycle request from one non-main canonical drive', async () => {
diff --git a/control-api/test/routes.gfsProxy.test.ts b/control-api/test/routes.gfsProxy.test.ts
index cb8053879..33abb5edd 100644
--- a/control-api/test/routes.gfsProxy.test.ts
+++ b/control-api/test/routes.gfsProxy.test.ts
@@ -208,6 +208,8 @@ describe('/api/v1/gfs/proxy', () => {
headers: {
'content-type': 'application/json',
'retry-after': '3',
+ 'x-ratelimit-limit': 'active_part_streams_global',
+ 'x-ratelimit-remaining': '0',
'upload-length': '209715200',
},
})
@@ -219,6 +221,8 @@ describe('/api/v1/gfs/proxy', () => {
expect(res.status).toBe(429)
expect(res.headers['retry-after']).toBe('3')
+ expect(res.headers['x-ratelimit-limit']).toBe('active_part_streams_global')
+ expect(res.headers['x-ratelimit-remaining']).toBe('0')
expect(res.headers['upload-length']).toBe('209715200')
})
diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsx
index 7b6174fb7..103df2733 100644
--- a/control-ui/components/__tests__/GfsBrowser.test.tsx
+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx
@@ -135,6 +135,7 @@ describe('GfsBrowser', () => {
mockGfsFetchFileBlob.mockReset()
mockCreateGfsUploadJob.mockClear()
mockUploadGfsFile.mockReset()
+ window.localStorage.clear()
mockUploadGfsFile.mockImplementation(async ({ file }: { file: File }) => {
if (file.size > GFS_FILE_UPLOAD_MAX_BYTES) {
throw new Error('GFS uploads are limited to 200 MB per file.')
@@ -355,6 +356,49 @@ describe('GfsBrowser', () => {
)
})
+ it('reuses the matching persisted session when a file is resumed through drag-and-drop', async () => {
+ const rootId = '11111111-1111-1111-1111-111111111111'
+ const rootRid = '11111111111111111111111111111111'
+ const uploadId = '55555555-5555-4555-8555-555555555555'
+ const lastModified = 1_725_000_000_000
+ window.localStorage.setItem(
+ 'evenfire:gfs-upload-v2:pending',
+ JSON.stringify({
+ uploadId,
+ fileName: 'resume.md',
+ fileSize: 11,
+ lastModified,
+ target: { operation: 'create', parentRid: rootRid },
+ name: 'resume.md',
+ })
+ )
+ mockApiGet.mockResolvedValue({
+ rootResourceId: rootId,
+ items: [],
+ nextCursor: null,
+ })
+ renderBrowser()
+ await screen.findByText('No resources are visible in this folder.')
+
+ const browser = screen.getByRole('region', { name: 'Global File System browser' })
+ const file = new File(['resume data'], 'resume.md', {
+ type: 'text/markdown',
+ lastModified,
+ })
+ const dataTransfer = { dropEffect: 'none', files: [file], types: ['Files'] }
+ fireEvent.drop(browser.querySelector('.cu-gfs-card')!, { dataTransfer })
+
+ await waitFor(() =>
+ expect(mockCreateGfsUploadJob).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file,
+ resumeUploadId: uploadId,
+ target: { operation: 'create', parentRid: rootRid },
+ })
+ )
+ )
+ })
+
it('shows byte progress and exposes pause/resume/cancel through the visible upload modal', async () => {
const rootId = '11111111-1111-1111-1111-111111111111'
const rootRid = '11111111111111111111111111111111'
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index 17f86ec52..99efa0e51 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -153,9 +153,14 @@ describe('GfsUploadJob', () => {
const method = init?.method ?? 'GET'
const url = String(_input)
if (method === 'GET' && url.endsWith('/capabilities'))
- return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
- status: 200,
- })
+ return new Response(
+ JSON.stringify({
+ upload: { resumableV2: { enabled: true } },
+ }),
+ {
+ status: 200,
+ }
+ )
if (method === 'POST' && url.endsWith('/uploads'))
return new Response(
JSON.stringify({
@@ -197,9 +202,12 @@ describe('GfsUploadJob', () => {
const method = init?.method ?? 'GET'
const url = String(_input)
if (method === 'GET' && url.endsWith('/capabilities'))
- return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
- status: 200,
- })
+ return new Response(
+ JSON.stringify({
+ upload: { resumableV2: { enabled: true } },
+ }),
+ { status: 200 }
+ )
if (method === 'POST' && url.endsWith('/uploads'))
return new Response(
JSON.stringify({
@@ -690,6 +698,78 @@ describe('GfsUploadJob', () => {
expect(completeCalls).toBe(1)
})
+ it('retries a missing target even when a sibling part still holds a lease', async () => {
+ const file = new File([new Uint8Array([5, 6])], 'active-sibling.bin')
+ const session = {
+ uploadId: '18181818-1818-4181-8181-181818181818',
+ drive: 'main',
+ expectedBytes: 2,
+ partBytes: 1,
+ partCount: 2,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ let statusCalls = 0
+ FakeXhr.responseLosses = 1
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(
+ JSON.stringify({
+ upload: { resumableV2: { enabled: true, maxConcurrentPartsPerSession: 1 } },
+ }),
+ { status: 200 }
+ )
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status')) {
+ statusCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ session: {
+ ...session,
+ state: 'uploading',
+ activePartCount: 1,
+ activePartNumbers: [1],
+ },
+ parts: [],
+ },
+ }),
+ { status: 200 }
+ )
+ }
+ if (method === 'POST' && url.endsWith('/complete'))
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: { ...session, state: 'completed', committedBytes: 2, committedPartCount: 2 },
+ }),
+ { status: 200 }
+ )
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-active-sibling' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(statusCalls).toBe(1)
+ expect(FakeXhr.requests).toHaveLength(3)
+ })
+
it('persists an unresolved response-loss session without replaying its part', async () => {
const file = new File([new Uint8Array([9, 10])], 'outcome-unknown.bin', {
lastModified: 14,
diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.ts
index 536b54d86..6ae738de7 100644
--- a/control-ui/lib/gfsFileUpload.ts
+++ b/control-ui/lib/gfsFileUpload.ts
@@ -9,6 +9,9 @@ import { apiSend } from './api'
const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'
const GFS_UPLOAD_DRIVE = 'main'
+// Retry only transient transport/edge failures. 507 (storage exhausted) and
+// other permanent 5xx responses are deliberately terminal: retrying them would
+// amplify a capacity incident instead of giving the operator a durable error.
export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3
@@ -60,6 +63,7 @@ export interface UploadReceipt {
committedBytes?: number
committedPartCount?: number
activePartCount: number
+ activePartNumbers?: number[]
expiresAt?: string
resultResourceId?: string
resultVersion?: number
@@ -592,8 +596,21 @@ async function reconcileAmbiguousPart(
part.partNumber,
new Error('writer returned an invalid activePartCount during reconciliation')
)
- if (activePartCount === 0) return 'missing'
- lastError = new Error(`writer still reports ${activePartCount} active parts`)
+ const activePartNumbers = status.session.activePartNumbers
+ if (activePartCount === 0 && activePartNumbers === undefined) return 'missing'
+ if (
+ !Array.isArray(activePartNumbers) ||
+ activePartNumbers.some(partNumber => !Number.isSafeInteger(partNumber) || partNumber < 0) ||
+ new Set(activePartNumbers).size !== activePartNumbers.length ||
+ activePartNumbers.length !== activePartCount
+ ) {
+ throw partOutcomeUnknown(
+ part.partNumber,
+ new Error('writer returned an invalid active part lease set during reconciliation')
+ )
+ }
+ if (!activePartNumbers.includes(part.partNumber)) return 'missing'
+ lastError = new Error(`writer still reports part ${part.partNumber} active`)
if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS) await delay(250 * 2 ** attempt, signal)
}
throw partOutcomeUnknown(part.partNumber, lastError)
diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.ts
index e6b7a7a10..6e07dd637 100644
--- a/desktop-app/src/__tests__/appService.gfs.test.ts
+++ b/desktop-app/src/__tests__/appService.gfs.test.ts
@@ -240,6 +240,7 @@ type UploadScopeTestService = {
operation: (token: string, signal: AbortSignal) => Promise<T>
) => Promise<T>
startGfsFileUpload: AppService['startGfsFileUpload']
+ startGfsFileReplace: AppService['startGfsFileReplace']
}
function authenticatedUploadService(statePath: string): UploadScopeTestService {
@@ -349,6 +350,29 @@ describe('AppService GFS upload security scope', () => {
}
})
+ it('never falls back to a legacy replace for an explicit resume', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-explicit-replace-resume-no-fallback-'))
+ try {
+ const filePath = join(root, 'replace.bin')
+ await writeFile(filePath, Buffer.from('replace payload'))
+ const service = authenticatedUploadService(join(root, 'gfs-upload-sessions.json'))
+ service.startDesktopGfsUpload.mockRejectedValue(
+ new DesktopUploadCapabilityError('resumable uploads are disabled')
+ )
+ service.gfsClient = {
+ createResource: vi.fn(),
+ replaceFile: vi.fn(),
+ }
+
+ await expect(
+ service.startGfsFileReplace(RESOURCE_ID, filePath, 'main', 3, 'replace-resume-upload-id')
+ ).rejects.toBeInstanceOf(DesktopUploadCapabilityError)
+ expect(service.gfsClient.replaceFile).not.toHaveBeenCalled()
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('keeps existing upload controls and read state available during a transient team hop', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-transient-controls-'))
try {
diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.ts
index 7ddaee2f3..71da03882 100644
--- a/desktop-app/src/gfs/upload.test.ts
+++ b/desktop-app/src/gfs/upload.test.ts
@@ -800,6 +800,93 @@ describe('desktop GFS indexed uploader', () => {
}
})
+ it('retries a missing target while status identifies a live sibling lease', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-active-sibling-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ await writeFile(filePath, Buffer.from([5, 6]))
+ const session = {
+ uploadId: '30303030-3030-4030-8030-303030303030',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: 2,
+ partBytes: 1,
+ partCount: 2,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let statusCalls = 0
+ let partCalls = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities({ maxConcurrentPartsPerSession: 1 }) as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') return {} as T
+ if (method === 'GET' && requestPath(url).endsWith('/status')) {
+ statusCalls += 1
+ return {
+ ok: true,
+ data: {
+ session: {
+ ...session,
+ state: 'uploading',
+ activePartCount: 1,
+ activePartNumbers: [1],
+ },
+ parts: [],
+ },
+ } as T
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete'))
+ return {
+ ok: true,
+ data: { ...session, state: 'completed', committedBytes: 2, committedPartCount: 2 },
+ } as T
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partCalls += 1
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* drain */
+ }
+ if (partCalls === 1) {
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ }
+ return { status: 204, text: '' }
+ },
+ }
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-active-sibling',
+ transport,
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(statusCalls).toBe(1)
+ expect(partCalls).toBe(3)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('never adopts or replays a response-loss part whose status identity differs', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-response-drift-'))
try {
diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.ts
index 5e510b8ca..cfa5cb67f 100644
--- a/desktop-app/src/gfs/upload.ts
+++ b/desktop-app/src/gfs/upload.ts
@@ -11,6 +11,9 @@ const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2
const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
+// Retry only transient transport/edge failures. 507 (storage exhausted) and
+// other permanent 5xx responses are deliberately terminal: retrying them would
+// amplify a capacity incident instead of giving the operator a durable error.
export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
@@ -35,6 +38,7 @@ export interface DesktopUploadSession {
committedBytes: number
committedPartCount: number
activePartCount: number
+ activePartNumbers?: number[]
expiresAt: string
resultResourceId?: string
resultVersion?: number
@@ -678,12 +682,21 @@ async function reconcileAmbiguousPart(
if (['aborted', 'canceling', 'failed', 'expired'].includes(status.session.state)) {
throw new Error(`cannot reconcile upload part in ${status.session.state}`)
}
- // activePartCount is the writer's durable lease fence. Zero plus an absent
- // committed row proves this part is safe to retry; a positive count cannot
- // identify which concurrent part owns the lease, so poll status rather than
- // risking an overlapping replay.
- if (status.session.activePartCount === 0) return 'missing'
- lastError = new Error(`writer still reports ${status.session.activePartCount} active parts`)
+ const activePartNumbers = status.session.activePartNumbers
+ if (status.session.activePartCount === 0 && activePartNumbers === undefined) return 'missing'
+ if (
+ !Array.isArray(activePartNumbers) ||
+ activePartNumbers.some(active => !Number.isSafeInteger(active) || active < 0) ||
+ new Set(activePartNumbers).size !== activePartNumbers.length ||
+ activePartNumbers.length !== status.session.activePartCount
+ ) {
+ throw partOutcomeUnknown(
+ partNumber,
+ new Error('writer returned an invalid active part lease set during reconciliation')
+ )
+ }
+ if (!activePartNumbers.includes(partNumber)) return 'missing'
+ lastError = new Error(`writer still reports part ${partNumber} active`)
if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS) await sleep(250 * 2 ** attempt)
}
throw partOutcomeUnknown(partNumber, lastError)
diff --git a/gfs-controller/src/api/envelope.ts b/gfs-controller/src/api/envelope.ts
index fd7fd4021..8980e6ac7 100644
--- a/gfs-controller/src/api/envelope.ts
+++ b/gfs-controller/src/api/envelope.ts
@@ -15,6 +15,8 @@ export interface ErrorBody {
code: string;
message: string;
reason?: string;
+ retryAfterSeconds?: number;
+ limit?: string;
}
export interface ErrEnvelope {
@@ -31,6 +33,8 @@ export function ok<T>(data: T): OkEnvelope<T> {
export function fail(error: GfsError): ErrEnvelope {
const body: ErrorBody = { code: error.code, message: error.message };
if (error.reason !== undefined) body.reason = error.reason;
+ if (error.retryAfterSeconds !== undefined) body.retryAfterSeconds = error.retryAfterSeconds;
+ if (error.limit !== undefined) body.limit = error.limit;
return { ok: false, error: body };
}
diff --git a/gfs-controller/src/api/errors.ts b/gfs-controller/src/api/errors.ts
index 2da687ae6..498bdff27 100644
--- a/gfs-controller/src/api/errors.ts
+++ b/gfs-controller/src/api/errors.ts
@@ -66,12 +66,21 @@ export type GoneReason = "resource_deleted" | "artifact_expired";
export class GfsError extends Error {
readonly code: GfsErrorCode;
readonly reason?: string;
+ readonly retryAfterSeconds?: number;
+ readonly limit?: string;
- constructor(code: GfsErrorCode, message: string, reason?: string) {
+ constructor(
+ code: GfsErrorCode,
+ message: string,
+ reason?: string,
+ details?: { retryAfterSeconds?: number; limit?: string },
+ ) {
super(message);
this.name = "GfsError";
this.code = code;
this.reason = reason;
+ this.retryAfterSeconds = details?.retryAfterSeconds;
+ this.limit = details?.limit;
}
get status(): number {
diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.ts
index 5e8160092..61d010187 100644
--- a/gfs-controller/src/api/serve.test.ts
+++ b/gfs-controller/src/api/serve.test.ts
@@ -21,6 +21,10 @@ class FakeRes extends Writable {
if (headers) Object.assign(this.headers, headers);
return this;
}
+ setHeader(name: string, value: string): this {
+ this.headers[name] = value;
+ return this;
+ }
_write(chunk: Buffer | string, _enc: BufferEncoding, cb: (e?: Error | null) => void): void {
this.chunks.push(Buffer.from(chunk));
cb();
@@ -268,6 +272,35 @@ describe("GFS Upload v2 route contract", () => {
expect(res.headers["Upload-Part-Number"]).toBe("0");
expect(res.headers["Upload-Checksum"]).toMatch(/^sha256 [A-Za-z0-9+/]{43}={1}$/);
});
+
+ it("exposes a bounded Retry-After and limit for writer quota responses", async () => {
+ const res = new FakeRes();
+ const quota = new GfsError(
+ "quota_exceeded",
+ "global upload stream concurrency limit reached",
+ undefined,
+ { retryAfterSeconds: 1, limit: "active_part_streams_global" },
+ );
+ await run(
+ uploadDeps({
+ uploadService: {
+ ...uploadDeps().uploadService,
+ get: async () => {
+ throw quota;
+ },
+ } as ServingDeps["uploadService"],
+ }),
+ req(`/v1/uploads/${UPLOAD_ID}`, { method: "HEAD", auth: "Bearer t" }),
+ res,
+ );
+ expect(res.statusCode).toBe(429);
+ expect(res.headers["Retry-After"]).toBe("1");
+ expect(res.headers["X-RateLimit-Limit"]).toBe("active_part_streams_global");
+ expect(res.json).toMatchObject({
+ ok: false,
+ error: { code: "quota_exceeded", retryAfterSeconds: 1, limit: "active_part_streams_global" },
+ });
+ });
});
describe("GfsReadServer.tryHandle — auth chain (fail-closed)", () => {
diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.ts
index 40f76d293..179933b77 100644
--- a/gfs-controller/src/api/serve.ts
+++ b/gfs-controller/src/api/serve.ts
@@ -310,7 +310,12 @@ export class GfsServingHandler {
)}`
);
}
- if (!res.headersSent) sendJson(res, status, body, abortConnectionHeader(req));
+ if (!res.headersSent) {
+ const retryAfter = body.error.retryAfterSeconds;
+ if (retryAfter !== undefined) res.setHeader("Retry-After", String(retryAfter));
+ if (body.error.limit !== undefined) res.setHeader("X-RateLimit-Limit", body.error.limit);
+ sendJson(res, status, body, abortConnectionHeader(req));
+ }
else res.end(); // a content stream already started; just terminate
} finally {
cancelCopyTimer?.();
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index 6ad8c90f9..c16872ece 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -139,6 +139,12 @@ class MemoryDb {
],
}
}
+ if (
+ sql.startsWith('SELECT upload_id FROM gfs_upload_sessions') &&
+ sql.includes("state = 'finalizing'")
+ ) {
+ return { rows: [] }
+ }
if (sql.startsWith('INSERT INTO gfs_upload_sessions')) {
const [
uploadId,
@@ -186,6 +192,7 @@ class MemoryDb {
failure_code: null,
expires_at: expiresAt,
completed_at: null,
+ finalizing_started_at: null,
cleanup_at: null,
}
this.sessions.push(row)
@@ -252,6 +259,11 @@ class MemoryDb {
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'uploading'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
row.state = 'uploading'
+ if (sql.includes('session_epoch = session_epoch + 1')) {
+ row.session_epoch = Number(row.session_epoch) + 1
+ row.finalizing_started_at = null
+ return { rows: [] }
+ }
row.active_part_count = Number(row.active_part_count) + 1
return { rows: [] }
}
@@ -269,7 +281,9 @@ class MemoryDb {
const row = this.sessions.find(session => session.upload_id === String(uploadId))!
row.committed_bytes = Number(row.committed_bytes) + Number(length)
row.committed_part_count = Number(row.committed_part_count) + 1
- row.active_part_count = Math.max(0, Number(row.active_part_count) - 1)
+ row.active_part_count = this.parts.filter(
+ part => part.upload_id === row.upload_id && part.state === 'reserved'
+ ).length
let contiguous = 0
for (let number = 0; number < Number(row.part_count); number += 1) {
const part = this.parts.find(
@@ -298,7 +312,9 @@ class MemoryDb {
}
if (sql.startsWith('UPDATE gfs_upload_sessions SET active_part_count')) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
- row.active_part_count = Math.max(0, Number(row.active_part_count) - 1)
+ row.active_part_count = this.parts.filter(
+ part => part.upload_id === row.upload_id && part.state === 'reserved'
+ ).length
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'paused'")) {
@@ -320,6 +336,7 @@ class MemoryDb {
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
row.state = 'finalizing'
+ row.finalizing_started_at = new Date().toISOString()
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'failed'")) {
@@ -327,6 +344,7 @@ class MemoryDb {
row.state = 'failed'
row.failure_code = values[1]
row.active_part_count = 0
+ row.finalizing_started_at = null
return { rows: [] }
}
if (sql.startsWith('UPDATE gfs_upload_sessions SET cleanup_at')) {
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index 89d9ddd7b..c2b3ac478 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -59,6 +59,8 @@ export interface UploadSessionReceipt {
committedBytes: number
committedPartCount: number
activePartCount: number
+ /** Reserved part numbers currently holding a durable lease. Status-only. */
+ activePartNumbers?: number[]
expiresAt: string
resultResourceId?: string
resultVersion?: number
@@ -119,6 +121,7 @@ export interface UploadSessionRow {
contiguousBytes: number
committedPartCount: number
activePartCount: number
+ finalizingStartedAt: string | null
sessionEpoch: number
state: string
resultResourceId: string | null
@@ -181,6 +184,10 @@ function rowToSession(row: Record<string, unknown>): UploadSessionRow {
contiguousBytes: Number(row.contiguous_bytes),
committedPartCount: Number(row.committed_part_count),
activePartCount: Number(row.active_part_count),
+ finalizingStartedAt:
+ row.finalizing_started_at == null
+ ? null
+ : new Date(String(row.finalizing_started_at)).toISOString(),
sessionEpoch: Number(row.session_epoch),
state: String(row.state),
resultResourceId: row.result_resource_id == null ? null : String(row.result_resource_id),
@@ -259,6 +266,13 @@ function requireEnabled(config: GfsUploadConfig): void {
if (!config.enabled) throw new GfsError('forbidden', 'resumable GFS uploads are disabled')
}
+function quotaExceeded(message: string, limit: string): GfsError {
+ return new GfsError('quota_exceeded', message, undefined, {
+ retryAfterSeconds: 1,
+ limit,
+ })
+}
+
function toReceipt(session: UploadSessionRow): UploadSessionReceipt {
return {
uploadId: session.uploadId,
@@ -576,7 +590,10 @@ export class GfsUploadSessionService {
[input.drive, input.ownerSubject]
)
if (Number(subjectActive.rows[0]?.count ?? 0) >= this.deps.config.maxActivePerSubject) {
- throw new GfsError('quota_exceeded', 'active upload session subject quota reached')
+ throw quotaExceeded(
+ 'active upload session subject quota reached',
+ 'active_sessions_subject'
+ )
}
const globalActive = await client.query(
`SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions
@@ -584,7 +601,10 @@ export class GfsUploadSessionService {
[]
)
if (Number(globalActive.rows[0]?.count ?? 0) >= this.deps.config.maxActiveGlobal) {
- throw new GfsError('quota_exceeded', 'active upload session global quota reached')
+ throw quotaExceeded(
+ 'active upload session global quota reached',
+ 'active_sessions_global'
+ )
}
if (available !== null) {
const reserved = await client.query(
@@ -660,6 +680,7 @@ export class GfsUploadSessionService {
)
const row = result.rows[0] ? rowToSession(result.rows[0]) : null
if (!row) throw new GfsError('not_found', 'upload session not found')
+ assertNotExpired(row, this.now())
if (row.state === 'expired') throw new GfsError('upload_expired', 'upload session has expired')
if (row.state === 'aborted') throw new GfsError('upload_aborted', 'upload session was canceled')
return toReceipt(row)
@@ -693,11 +714,18 @@ export class GfsUploadSessionService {
FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' AND part_number > $2 ORDER BY part_number ASC LIMIT $3`,
[uploadId, cursor, requestedLimit + 1]
)
+ const active = await this.deps.db.query(
+ `SELECT part_number FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'reserved' ORDER BY part_number ASC`,
+ [uploadId]
+ )
const hasMore = result.rows.length > requestedLimit
const rows = hasMore ? result.rows.slice(0, requestedLimit) : result.rows
const lastPart = rows.at(-1)
return {
- session: toReceipt(session),
+ session: {
+ ...toReceipt(session),
+ activePartNumbers: active.rows.map(row => Number(row.part_number)),
+ },
parts: rows.map(row => {
const part = rowToPart(row)
return {
@@ -792,10 +820,16 @@ export class GfsUploadSessionService {
Number(activeGlobal.rows[0]?.count ?? 0) >=
this.deps.config.maxConcurrentPartStreamsGlobal
) {
- throw new GfsError('quota_exceeded', 'global upload stream concurrency limit reached')
+ throw quotaExceeded(
+ 'global upload stream concurrency limit reached',
+ 'active_part_streams_global'
+ )
}
if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession)
- throw new GfsError('quota_exceeded', 'per-session part concurrency limit reached')
+ throw quotaExceeded(
+ 'per-session part concurrency limit reached',
+ 'active_part_streams_session'
+ )
const stagingPath = sessionPath(
this.deps.storageMountPath,
uploadId,
@@ -839,7 +873,9 @@ export class GfsUploadSessionService {
)
}
await client.query(
- `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = active_part_count + 1, updated_at = now()
+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = (
+ SELECT COUNT(*) FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'reserved'
+ ), updated_at = now()
WHERE upload_id = $1`,
[uploadId]
)
@@ -968,7 +1004,9 @@ export class GfsUploadSessionService {
`UPDATE gfs_upload_sessions
SET committed_bytes = committed_bytes + $2,
committed_part_count = committed_part_count + 1,
- active_part_count = GREATEST(active_part_count - 1, 0),
+ active_part_count = (
+ SELECT COUNT(*) FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'reserved'
+ ),
contiguous_bytes = COALESCE((
SELECT SUM(p.length_bytes) FROM gfs_upload_parts p
WHERE p.upload_id = $1 AND p.state = 'committed'
@@ -1047,7 +1085,9 @@ export class GfsUploadSessionService {
// reconciliation would undercount other concurrent parts.
if (failedPart.rows.length > 0) {
await client.query(
- `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()
+ `UPDATE gfs_upload_sessions SET active_part_count = (
+ SELECT COUNT(*) FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'reserved'
+ ), updated_at = now()
WHERE upload_id = $1`,
[uploadId]
)
@@ -1252,13 +1292,39 @@ export class GfsUploadSessionService {
)
if (failed.rows.length === 1) {
await client.query(
- `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()
- WHERE upload_id = $1`,
+ `UPDATE gfs_upload_sessions SET active_part_count = (
+ SELECT COUNT(*) FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'reserved'
+ ), updated_at = now()
+ WHERE upload_id = $1`,
[row.upload_id]
)
}
if (failed.rows.length === 1) staleParts += 1
}
+ const staleFinalizers = await client.query(
+ `SELECT upload_id
+ FROM gfs_upload_sessions
+ WHERE state = 'finalizing'
+ AND finalizing_started_at IS NOT NULL
+ AND finalizing_started_at < now() - ($1::bigint * interval '1 millisecond')
+ LIMIT 100
+ FOR UPDATE SKIP LOCKED`,
+ [this.deps.config.finalizeTimeoutMs]
+ )
+ for (const row of staleFinalizers.rows) {
+ const uploadId = String(row.upload_id)
+ // A live finalizer owns the in-process fence. A different writer may
+ // still finish against the same row, so the session epoch update below
+ // is the durable fence that makes any late publication fail closed.
+ if (this.inFlightFinalizers.has(uploadId)) continue
+ await client.query(
+ `UPDATE gfs_upload_sessions
+ SET state = 'uploading', session_epoch = session_epoch + 1,
+ finalizing_started_at = NULL, updated_at = now()
+ WHERE upload_id = $1 AND state = 'finalizing'`,
+ [uploadId]
+ )
+ }
const expired = await client.query(
`SELECT upload_id FROM gfs_upload_sessions
WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()
@@ -1468,7 +1534,10 @@ export class GfsUploadSessionService {
[]
)
if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {
- throw new GfsError('quota_exceeded', 'upload finalization concurrency limit reached')
+ throw quotaExceeded(
+ 'upload finalization concurrency limit reached',
+ 'active_finalizations'
+ )
}
if (!locked.wholeSha256 && expectedWhole) {
await client.query(
@@ -1477,7 +1546,7 @@ export class GfsUploadSessionService {
)
}
await client.query(
- `UPDATE gfs_upload_sessions SET state = 'finalizing', updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,
+ `UPDATE gfs_upload_sessions SET state = 'finalizing', finalizing_started_at = now(), updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,
[uploadId, locked.sessionEpoch]
)
return {
@@ -1523,7 +1592,8 @@ export class GfsUploadSessionService {
if (locked.state === 'finalizing' && locked.sessionEpoch === started.sessionEpoch) {
await client.query(
`UPDATE gfs_upload_sessions
- SET state = 'failed', failure_code = $2, active_part_count = 0, updated_at = now()
+ SET state = 'failed', failure_code = $2, active_part_count = 0,
+ finalizing_started_at = NULL, updated_at = now()
WHERE upload_id = $1`,
[uploadId, failureCode]
)
@@ -1564,7 +1634,7 @@ export class GfsUploadSessionService {
if (locked.state !== 'finalizing' || locked.sessionEpoch !== started.sessionEpoch)
throw new GfsError('upload_aborted', 'upload session changed during finalization')
await client.query(
- `UPDATE gfs_upload_sessions SET state = 'completed', result_resource_id = $2, result_version = $3, result_sha256 = $4, completed_at = now(), updated_at = now() WHERE upload_id = $1`,
+ `UPDATE gfs_upload_sessions SET state = 'completed', result_resource_id = $2, result_version = $3, result_sha256 = $4, completed_at = now(), finalizing_started_at = NULL, updated_at = now() WHERE upload_id = $1`,
[uploadId, receipt.resourceId, receipt.version, receipt.sha256]
)
return {
diff --git a/scripts/tests/test-gfs-upload-v2-geometry-contract.sh b/scripts/tests/test-gfs-upload-v2-geometry-contract.sh
index b76ba3223..eec24ce04 100755
--- a/scripts/tests/test-gfs-upload-v2-geometry-contract.sh
+++ b/scripts/tests/test-gfs-upload-v2-geometry-contract.sh
@@ -15,14 +15,29 @@ read_const() {
gfsc_preferred="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"
gfsc_max="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"
+gfsc_concurrency="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_DEFAULT_CONCURRENCY)"
+gfsc_fallback="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_FALLBACK_CONCURRENCY)"
+gfsc_threshold="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_INSTABILITY_FAILURE_THRESHOLD)"
ui_preferred="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_PREFERRED_PART_BYTES)"
ui_max="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_MAX_PART_BYTES)"
+ui_concurrency="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_DEFAULT_CONCURRENCY)"
+ui_fallback="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY)"
+ui_threshold="$(read_const control-ui/lib/gfsFileUpload.ts GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD)"
desktop_preferred="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"
desktop_max="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"
+desktop_concurrency="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_DEFAULT_CONCURRENCY)"
+desktop_fallback="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_FALLBACK_CONCURRENCY)"
+desktop_threshold="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD)"
[[ -n "$gfsc_preferred" && "$gfsc_preferred" == "$ui_preferred" && "$gfsc_preferred" == "$desktop_preferred" ]] \
|| fail "preferred part geometry diverges between writer, Control UI, and Desktop"
[[ -n "$gfsc_max" && "$gfsc_max" == "$ui_max" && "$gfsc_max" == "$desktop_max" ]] \
|| fail "maximum part geometry diverges between writer, Control UI, and Desktop"
+[[ -n "$gfsc_concurrency" && "$gfsc_concurrency" == "$ui_concurrency" && "$gfsc_concurrency" == "$desktop_concurrency" ]] \
+ || fail "default concurrency diverges between writer, Control UI, and Desktop"
+[[ -n "$gfsc_fallback" && "$gfsc_fallback" == "$ui_fallback" && "$gfsc_fallback" == "$desktop_fallback" ]] \
+ || fail "fallback concurrency diverges between writer, Control UI, and Desktop"
+[[ -n "$gfsc_threshold" && "$gfsc_threshold" == "$ui_threshold" && "$gfsc_threshold" == "$desktop_threshold" ]] \
+ || fail "instability failure threshold diverges between writer, Control UI, and Desktop"
echo "PASS: GFS Upload v2 part geometry is aligned across writer, Control UI, and Desktop"