Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs-upload): reconcile ambiguous part responses safely

Alfredo Lopez52f40160aaa9
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index fafbc3261..d21787b80 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -17,6 +17,7 @@ describe('assertGfsFileUploadSize', () => {
class FakeXhr {
static requests: FakeXhr[] = []
static statuses: number[] = []
+ static responseLosses = 0
upload: { onprogress: ((event: { lengthComputable: boolean; loaded: number }) => void) | null } =
{ onprogress: null }
status = 204
@@ -41,6 +42,11 @@ class FakeXhr {
loaded: Math.max(1, Math.floor(body.size / 2)),
})
this.upload.onprogress?.({ lengthComputable: true, loaded: body.size })
+ if (FakeXhr.responseLosses > 0) {
+ FakeXhr.responseLosses -= 1
+ this.onerror?.()
+ return
+ }
this.onload?.()
})
}
@@ -53,6 +59,7 @@ describe('GfsUploadJob', () => {
beforeEach(() => {
FakeXhr.requests = []
FakeXhr.statuses = []
+ FakeXhr.responseLosses = 0
vi.stubGlobal('XMLHttpRequest', FakeXhr)
})
@@ -64,7 +71,7 @@ describe('GfsUploadJob', () => {
it('resumes only the missing part and emits visible progress before completion', async () => {
const file = new File([new Uint8Array([1, 2, 3, 4])], 'payload.bin', { lastModified: 12 })
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', new Uint8Array([1, 2])))
- const part0 = btoa(String.fromCharCode(...digest))
+ const part0 = [...digest].map(value => value.toString(16).padStart(2, '0')).join('')
const responses: Array<{ status: number; body?: unknown }> = []
const status: GfsUploadStatus = {
session: {
@@ -121,11 +128,236 @@ describe('GfsUploadJob', () => {
expect(result.state).toBe('completed')
expect(FakeXhr.requests).toHaveLength(1)
expect(FakeXhr.requests[0]?.headers.get('Upload-Part-Number')).toBe('1')
+ const part1Digest = new Uint8Array(
+ await crypto.subtle.digest('SHA-256', new Uint8Array([3, 4]))
+ )
+ expect(FakeXhr.requests[0]?.headers.get('Upload-Checksum')).toBe(
+ `sha256 ${btoa(String.fromCharCode(...part1Digest))}`
+ )
expect(progress.some(value => value < 4)).toBe(true)
expect(progress.at(-1)).toBe(4)
expect(responses).toHaveLength(0)
})
+ it('adopts a status-confirmed committed part after its XHR response is lost', async () => {
+ const file = new File([new Uint8Array([5, 6, 7, 8])], 'response-loss.bin', {
+ lastModified: 13,
+ })
+ const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', await file.arrayBuffer()))
+ const sha256 = [...digest].map(value => value.toString(16).padStart(2, '0')).join('')
+ const uploadId = '15151515-1515-4151-8151-151515151515'
+ const session = {
+ uploadId,
+ expectedBytes: file.size,
+ partBytes: file.size,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ let statusCalls = 0
+ let completeCalls = 0
+ FakeXhr.responseLosses = 1
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status')) {
+ statusCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ session: {
+ ...session,
+ state: 'uploading',
+ contiguousBytes: file.size,
+ committedBytes: file.size,
+ committedPartCount: 1,
+ },
+ parts: [
+ {
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: file.size,
+ sha256,
+ },
+ ],
+ },
+ }),
+ { status: 200 }
+ )
+ }
+ if (method === 'POST' && url.endsWith('/complete')) {
+ completeCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ ...session,
+ state: 'completed',
+ contiguousBytes: file.size,
+ committedBytes: file.size,
+ committedPartCount: 1,
+ },
+ }),
+ { status: 200 }
+ )
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-response-loss' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+
+ expect(FakeXhr.requests).toHaveLength(1)
+ expect(statusCalls).toBe(1)
+ expect(completeCalls).toBe(1)
+ })
+
+ it('persists an unresolved response-loss session without replaying its part', async () => {
+ const file = new File([new Uint8Array([9, 10])], 'outcome-unknown.bin', {
+ lastModified: 14,
+ })
+ const session = {
+ uploadId: '16161616-1616-4161-8161-161616161616',
+ expectedBytes: file.size,
+ partBytes: file.size,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ let reconcileStatusCalls = 0
+ let completeCalls = 0
+ let persisted = 0
+ FakeXhr.responseLosses = 1
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status')) {
+ reconcileStatusCalls += 1
+ throw new Error('status transport unavailable')
+ }
+ if (method === 'POST' && url.endsWith('/complete')) {
+ completeCalls += 1
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 200 })
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-outcome-unknown' },
+ onPersist: () => {
+ persisted += 1
+ },
+ }).start()
+ ).rejects.toThrow('One or more upload parts could not be committed.')
+
+ expect(persisted).toBe(1)
+ expect(FakeXhr.requests).toHaveLength(1)
+ expect(reconcileStatusCalls).toBe(3)
+ expect(completeCalls).toBe(0)
+ })
+
+ it('fails closed when response-loss status omits or corrupts activePartCount', async () => {
+ const file = new File([new Uint8Array([11, 12])], 'invalid-active-count.bin', {
+ lastModified: 15,
+ })
+ const invalidCounts: unknown[] = [undefined, 0.5, -1]
+
+ for (const [index, activePartCount] of invalidCounts.entries()) {
+ const session = {
+ uploadId: `17171717-1717-4171-8171-17171717171${index}`,
+ expectedBytes: file.size,
+ partBytes: file.size,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ ...(activePartCount === undefined ? {} : { activePartCount }),
+ }
+ let statusCalls = 0
+ let completeCalls = 0
+ FakeXhr.requests = []
+ FakeXhr.statuses = []
+ FakeXhr.responseLosses = 1
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status')) {
+ statusCalls += 1
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: { session: { ...session, state: 'uploading' }, parts: [] },
+ }),
+ { status: 200 }
+ )
+ }
+ if (method === 'POST' && url.endsWith('/complete')) {
+ completeCalls += 1
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 200 })
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: `parent-invalid-active-${index}` },
+ }).start()
+ ).rejects.toThrow('One or more upload parts could not be committed.')
+
+ expect(FakeXhr.requests).toHaveLength(1)
+ expect(statusCalls).toBe(1)
+ expect(completeCalls).toBe(0)
+ }
+ })
+
it('rejects a committed status part whose checksum does not match the selected file before PUT', async () => {
const file = new File([new Uint8Array([1, 2])], 'payload.bin')
vi.stubGlobal(
@@ -166,7 +398,7 @@ describe('GfsUploadJob', () => {
target: { operation: 'create', parentRid: 'parent-2' },
resumeUploadId: '22222222-2222-4222-8222-222222222222',
}).start()
- ).rejects.toThrow(/does not match/)
+ ).rejects.toThrow(/invalid SHA-256|does not match/)
expect(FakeXhr.requests).toHaveLength(0)
})
@@ -213,6 +445,28 @@ describe('GfsUploadJob', () => {
{ status: 201 }
)
}
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && String(_input).includes('/status'))
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ session: {
+ uploadId: '33333333-3333-4333-8333-333333333333',
+ expectedBytes: 8,
+ partBytes: 2,
+ partCount: 4,
+ state: 'uploading',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ },
+ parts: [],
+ },
+ }),
+ { status: 200 }
+ )
if (method === 'POST' && String(_input).endsWith('/complete')) {
completeCalls += 1
return new Response(
diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.ts
index 9e4930744..dec63d25f 100644
--- a/control-ui/lib/gfsFileUpload.ts
+++ b/control-ui/lib/gfsFileUpload.ts
@@ -9,6 +9,9 @@ import { apiSend } from './api'
const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'
const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
+const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000
+const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60_000
+const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
export interface GfsUploadTarget {
operation: 'create' | 'replace'
@@ -35,7 +38,7 @@ export interface UploadReceipt {
contiguousBytes?: number
committedBytes?: number
committedPartCount?: number
- activePartCount?: number
+ activePartCount: number
expiresAt?: string
resultResourceId?: string
resultVersion?: number
@@ -168,9 +171,10 @@ async function requestJson<T>(
method: 'GET' | 'POST' | 'HEAD' | 'DELETE',
path: string,
body?: unknown,
- signal?: AbortSignal
+ signal?: AbortSignal,
+ timeoutMs = 600_000
): Promise<{ data: T; response: Response }> {
- const timed = signalWithTimeout(signal, 600_000)
+ const timed = signalWithTimeout(signal, timeoutMs)
try {
const response = await fetch(endpoint(path), {
method,
@@ -221,7 +225,14 @@ async function uploadError(
}
function assertRetryable(error: unknown): boolean {
- if (error instanceof DOMException && error.name === 'AbortError') return false
+ if (
+ error instanceof Error &&
+ (error.name === 'AbortError' ||
+ error.message.startsWith('source_changed:') ||
+ error.message.startsWith('upload_part_mismatch:') ||
+ error.message.startsWith('upload_part_outcome_unknown:'))
+ )
+ return false
const status =
typeof error === 'object' && error !== null ? (error as { status?: unknown }).status : undefined
return typeof status === 'number' ? RETRYABLE_STATUS.has(status) : true
@@ -253,17 +264,185 @@ function digestBase64(bytes: ArrayBuffer): string {
return btoa(binary)
}
+function digestHex(bytes: ArrayBuffer): string {
+ return [...new Uint8Array(bytes)].map(value => value.toString(16).padStart(2, '0')).join('')
+}
+
+interface GfsPartChecksum {
+ base64: string
+ hex: string
+}
+
+async function sha256Part(blob: Blob): Promise<GfsPartChecksum> {
+ const digest = await crypto.subtle.digest('SHA-256', await blob.arrayBuffer())
+ return { base64: digestBase64(digest), hex: digestHex(digest) }
+}
+
+function indexCommittedParts(
+ session: UploadReceipt,
+ parts: readonly GfsUploadPartReceipt[]
+): Map<number, GfsUploadPartReceipt> {
+ const committed = new Map<number, GfsUploadPartReceipt>()
+ for (const part of parts) {
+ if (
+ !Number.isSafeInteger(part.partNumber) ||
+ part.partNumber < 0 ||
+ part.partNumber >= session.partCount
+ )
+ throw new Error(
+ `upload_part_mismatch: part number ${part.partNumber} is outside the upload geometry`
+ )
+ const expectedOffset = part.partNumber * session.partBytes
+ const expectedLength = Math.min(session.partBytes, session.expectedBytes - expectedOffset)
+ if (part.offsetBytes !== expectedOffset || part.lengthBytes !== expectedLength)
+ throw new Error(
+ `upload_part_mismatch: part ${part.partNumber} does not match the upload geometry`
+ )
+ if (committed.has(part.partNumber))
+ throw new Error(`upload_part_mismatch: part ${part.partNumber} was returned more than once`)
+ if (!/^[0-9a-f]{64}$/i.test(part.sha256))
+ throw new Error(`upload_part_mismatch: part ${part.partNumber} has an invalid SHA-256 digest`)
+ committed.set(part.partNumber, part)
+ }
+ return committed
+}
+
+async function loadUploadStatus(
+ uploadId: string,
+ expected: { expectedBytes: number; partBytes?: number; partCount?: number },
+ signal?: AbortSignal,
+ timeoutMs = 600_000
+): Promise<{ status: GfsUploadStatus; head: Response }> {
+ const head = await requestJson<undefined>(
+ 'HEAD',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}`,
+ undefined,
+ signal,
+ timeoutMs
+ )
+ const declaredLengthHeader = head.response.headers.get('upload-length')
+ if (declaredLengthHeader !== null) {
+ const declaredLength = Number(declaredLengthHeader)
+ if (Number.isSafeInteger(declaredLength) && declaredLength !== expected.expectedBytes)
+ throw new Error('selected file size differs from the persisted upload session')
+ }
+
+ let cursor: string | undefined
+ let session: UploadReceipt | null = null
+ const parts: GfsUploadPartReceipt[] = []
+ for (let page = 0; page < 8; page += 1) {
+ const query = cursor ? `?limit=256&cursor=${encodeURIComponent(cursor)}` : '?limit=256'
+ const response = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
+ 'GET',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}/status${query}`,
+ undefined,
+ signal,
+ timeoutMs
+ )
+ const current = receiptFromResponse(response)
+ session ??= current.session
+ if (
+ session.uploadId !== current.session.uploadId ||
+ session.uploadId !== uploadId ||
+ session.expectedBytes !== current.session.expectedBytes ||
+ session.expectedBytes !== expected.expectedBytes ||
+ session.partBytes !== current.session.partBytes ||
+ session.partCount !== current.session.partCount ||
+ (expected.partBytes !== undefined && current.session.partBytes !== expected.partBytes) ||
+ (expected.partCount !== undefined && current.session.partCount !== expected.partCount)
+ )
+ throw new Error('upload status changed during reconciliation')
+ parts.push(...current.parts)
+ if (!current.nextCursor)
+ return { status: { session, parts, nextCursor: null }, head: head.response }
+ cursor = current.nextCursor
+ if (page === 7) throw new Error('upload status pagination exceeded the bounded page limit')
+ }
+ throw new Error('upload status pagination did not terminate')
+}
+
+function ambiguousPartResponse(status: number | undefined): boolean {
+ return status === undefined || status === 408 || status >= 500
+}
+
+function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {
+ return new Error(
+ `upload_part_outcome_unknown: could not prove whether part ${partNumber} committed; resume the persisted session`,
+ { cause }
+ )
+}
+
+async function reconcileAmbiguousPart(
+ file: File,
+ session: UploadReceipt,
+ part: { partNumber: number; offsetBytes: number; lengthBytes: number },
+ checksum: GfsPartChecksum,
+ signal?: AbortSignal
+): Promise<'committed' | 'missing'> {
+ let lastError: unknown
+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {
+ let status: GfsUploadStatus
+ try {
+ status = (
+ await loadUploadStatus(
+ session.uploadId,
+ {
+ expectedBytes: file.size,
+ partBytes: session.partBytes,
+ partCount: session.partCount,
+ },
+ signal,
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS
+ )
+ ).status
+ } catch (error) {
+ if (signal?.aborted) throw signal.reason ?? error
+ lastError = error
+ if (!assertRetryable(error) || attempt + 1 >= GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)
+ throw partOutcomeUnknown(part.partNumber, error)
+ await delay(250 * 2 ** attempt, signal)
+ continue
+ }
+
+ const committed = indexCommittedParts(status.session, status.parts).get(part.partNumber)
+ if (committed) {
+ if (committed.offsetBytes !== part.offsetBytes || committed.lengthBytes !== part.lengthBytes)
+ throw new Error(
+ `upload_part_mismatch: part ${part.partNumber} does not match the upload geometry`
+ )
+ if (committed.sha256.toLowerCase() !== checksum.hex)
+ throw new Error(
+ `source_changed: committed part ${part.partNumber} differs from the selected file`
+ )
+ return 'committed'
+ }
+
+ if (['aborted', 'canceling', 'failed', 'expired'].includes(status.session.state))
+ throw new Error(`cannot reconcile upload part in ${status.session.state}`)
+ const activePartCount = status.session.activePartCount
+ if (!Number.isSafeInteger(activePartCount) || activePartCount < 0)
+ throw partOutcomeUnknown(
+ part.partNumber,
+ new Error('writer returned an invalid activePartCount during reconciliation')
+ )
+ if (activePartCount === 0) return 'missing'
+ lastError = new Error(`writer still reports ${activePartCount} active parts`)
+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS) await delay(250 * 2 ** attempt, signal)
+ }
+ throw partOutcomeUnknown(part.partNumber, lastError)
+}
+
async function putPart(
uploadId: string,
file: File,
part: { partNumber: number; offsetBytes: number; lengthBytes: number },
partCount: number,
+ blob: Blob,
+ checksumBase64: string,
signal: AbortSignal | undefined,
onByteProgress?: (progress: GfsUploadProgress) => void
): Promise<void> {
- const blob = file.slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)
- const checksum = digestBase64(await crypto.subtle.digest('SHA-256', await blob.arrayBuffer()))
- const timed = signalWithTimeout(signal, 600_000)
+ const timed = signalWithTimeout(signal, GFS_UPLOAD_V2_PART_TIMEOUT_MS)
try {
await new Promise<void>((resolve, reject) => {
const xhr = new XMLHttpRequest()
@@ -279,7 +458,7 @@ async function putPart(
xhr.setRequestHeader('Upload-Part-Number', String(part.partNumber))
xhr.setRequestHeader('Upload-Offset', String(part.offsetBytes))
xhr.setRequestHeader('Upload-Chunk-Length', String(part.lengthBytes))
- xhr.setRequestHeader('Upload-Checksum', `sha256 ${checksum}`)
+ xhr.setRequestHeader('Upload-Checksum', `sha256 ${checksumBase64}`)
xhr.upload.onprogress = event => {
if (!event.lengthComputable) return
onByteProgress?.({
@@ -301,7 +480,12 @@ async function putPart(
reject(error)
}
xhr.onerror = () => reject(new Error('GFS upload network error'))
- xhr.onabort = () => reject(abortError('GFS upload request aborted'))
+ xhr.onabort = () =>
+ reject(
+ signal?.aborted
+ ? abortError('GFS upload request aborted')
+ : new Error('GFS upload response timed out')
+ )
timed.signal.addEventListener('abort', abort, { once: true })
xhr.send(blob)
})
@@ -311,7 +495,7 @@ async function putPart(
}
async function uploadPartWithRetries(
- uploadId: string,
+ session: UploadReceipt,
file: File,
part: { partNumber: number; offsetBytes: number; lengthBytes: number },
partCount: number,
@@ -320,16 +504,35 @@ async function uploadPartWithRetries(
onRetryableFailure?: () => void,
onPartSuccess?: () => void
): Promise<void> {
+ const blob = file.slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)
+ const checksum = await sha256Part(blob)
let lastError: unknown
for (let attempt = 0; attempt < 3; attempt += 1) {
try {
- await putPart(uploadId, file, part, partCount, signal, onByteProgress)
+ await putPart(
+ session.uploadId,
+ file,
+ part,
+ partCount,
+ blob,
+ checksum.base64,
+ signal,
+ onByteProgress
+ )
onPartSuccess?.()
return
} catch (error) {
lastError = error
+ if (signal?.aborted) throw signal.reason ?? error
if (!assertRetryable(error)) throw error
onRetryableFailure?.()
+ if (ambiguousPartResponse(statusOf(error))) {
+ const reconciled = await reconcileAmbiguousPart(file, session, part, checksum, signal)
+ if (reconciled === 'committed') {
+ onPartSuccess?.()
+ return
+ }
+ }
if (attempt === 2) throw error
await delay(Math.min(1000, 150 * (attempt + 1)), signal)
}
@@ -338,7 +541,7 @@ async function uploadPartWithRetries(
}
async function runParts(
- uploadId: string,
+ session: UploadReceipt,
file: File,
parts: Array<{ partNumber: number; offsetBytes: number; lengthBytes: number }>,
concurrency: number,
@@ -386,7 +589,7 @@ async function runParts(
const part = work[index]!
try {
await uploadPartWithRetries(
- uploadId,
+ session,
file,
part,
parts.length,
@@ -626,44 +829,17 @@ export class GfsUploadJob {
const resumable = upload.resumableV2!
const resumeId = this.session?.uploadId ?? this.input.resumeUploadId
if (resumeId) {
- const head = await requestJson<undefined>(
- 'HEAD',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(resumeId)}`,
- undefined,
- this.abortController.signal
- )
- const length = Number(head.response.headers.get('upload-length'))
- if (Number.isSafeInteger(length) && length !== this.input.file.size)
- throw new Error('selected file size differs from the persisted upload session')
- let cursor: string | undefined
- let session: UploadReceipt | null = null
- const parts: GfsUploadPartReceipt[] = []
- for (let page = 0; page < 8; page += 1) {
- const query = cursor ? `?limit=256&cursor=${encodeURIComponent(cursor)}` : '?limit=256'
- const statusResponse = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
- 'GET',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(resumeId)}/status${query}`,
- undefined,
+ const { session, parts } = (
+ await loadUploadStatus(
+ resumeId,
+ { expectedBytes: this.input.file.size },
this.abortController.signal
)
- const status = receiptFromResponse(statusResponse)
- session ??= status.session
- if (
- session.uploadId !== status.session.uploadId ||
- session.expectedBytes !== status.session.expectedBytes
- )
- throw new Error('upload status changed during resume')
- parts.push(...status.parts)
- if (!status.nextCursor) break
- cursor = status.nextCursor
- if (page === 7) throw new Error('upload status pagination exceeded the bounded page limit')
- }
- if (!session || session.expectedBytes !== this.input.file.size)
- throw new Error('selected file size differs from the upload session')
+ ).status
if (['completed', 'aborted', 'failed'].includes(session.state))
return { session, committed: new Set(), resumable }
- await this.validateCommittedParts(session, parts)
- return { session, committed: new Set(parts.map(part => part.partNumber)), resumable }
+ const committed = await this.validateCommittedParts(session, parts)
+ return { session, committed, resumable }
}
const createBody = {
operation: this.input.target.operation,
@@ -709,29 +885,16 @@ export class GfsUploadJob {
private async validateCommittedParts(
session: UploadReceipt,
parts: GfsUploadPartReceipt[]
- ): Promise<void> {
- for (const part of parts) {
- const expectedOffset = part.partNumber * session.partBytes
- const expectedLength = Math.min(session.partBytes, this.input.file.size - expectedOffset)
- if (
- part.offsetBytes !== expectedOffset ||
- part.lengthBytes !== expectedLength ||
- part.partNumber < 0 ||
- part.partNumber >= session.partCount
- ) {
- throw new Error('upload status contains invalid part geometry')
- }
- const checksum = digestBase64(
- await crypto.subtle.digest(
- 'SHA-256',
- await this.input.file
- .slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)
- .arrayBuffer()
- )
+ ): Promise<Set<number>> {
+ const committed = indexCommittedParts(session, parts)
+ for (const part of committed.values()) {
+ const checksum = await sha256Part(
+ this.input.file.slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)
)
- if (checksum !== part.sha256)
+ if (checksum.hex !== part.sha256.toLowerCase())
throw new Error(`upload part ${part.partNumber} does not match the selected file`)
}
+ return new Set(committed.keys())
}
private async run(): Promise<UploadReceipt> {
@@ -797,7 +960,7 @@ export class GfsUploadJob {
const fallbackConcurrency =
Number(prepared.resumable.fallbackConcurrency) || GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY
const result = await runParts(
- prepared.session.uploadId,
+ prepared.session,
this.input.file,
pending,
concurrency,
diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.ts
index 43e49d5ce..6a0a205b3 100644
--- a/desktop-app/src/gfs/upload.test.ts
+++ b/desktop-app/src/gfs/upload.test.ts
@@ -13,6 +13,10 @@ function expectCanonicalDrive(url: string, drive: string): void {
expect(new URL(url).searchParams.get('drive')).toBe(drive)
}
+function enabledCapabilities(overrides: Record<string, unknown> = {}) {
+ return { upload: { resumableV2: { enabled: true, ...overrides } } }
+}
+
describe('desktop GFS indexed uploader', () => {
it('streams independent binary parts with base64 checksums and completes the session', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-'))
@@ -21,13 +25,11 @@ describe('desktop GFS indexed uploader', () => {
const bytes = Buffer.concat([Buffer.alloc(1024 * 1024, 0x11), Buffer.alloc(512 * 1024, 0x22)])
await writeFile(filePath, bytes)
const partRequests: Array<{ headers: Record<string, string>; body: Buffer }> = []
+ const snapshots: number[] = []
const transport = {
async requestJson<T>(method: 'GET' | 'POST' | 'DELETE'): Promise<T> {
if (method === 'GET') {
- return {
- ok: true,
- data: { upload: { resumableV2: { enabled: true, maxFileBytes: 200 * 1024 * 1024 } } },
- } as T
+ return enabledCapabilities({ maxFileBytes: 200 * 1024 * 1024 }) as T
}
if (method === 'POST') {
return {
@@ -73,8 +75,12 @@ describe('desktop GFS indexed uploader', () => {
operation: 'create',
parentRid: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
transport,
+ onState: snapshot => snapshots.push(snapshot.uploadedBytes),
})
expect(result.state).toBe('initiated')
+ expect(snapshots[0]).toBe(0)
+ expect(new Set(snapshots).size).toBeGreaterThanOrEqual(3)
+ expect(snapshots).toContain(bytes.length)
expect(partRequests).toHaveLength(2)
for (const request of partRequests) {
const expected = createHash('sha256').update(request.body).digest('base64')
@@ -90,6 +96,61 @@ describe('desktop GFS indexed uploader', () => {
}
})
+ it('rejects malformed or enveloped capability payloads before creating a session', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-capabilities-contract-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ await writeFile(filePath, Buffer.from([0x01]))
+ const driftCases = [
+ {
+ label: 'obsolete envelope',
+ response: { ok: true, data: enabledCapabilities() },
+ cause: 'missing upload object',
+ },
+ {
+ label: 'malformed enabled flag',
+ response: { upload: { resumableV2: { enabled: 'true' } } },
+ cause: 'resumableV2.enabled must be boolean',
+ },
+ ]
+
+ for (const drift of driftCases) {
+ let sessionRequests = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities')) {
+ return drift.response as T
+ }
+ sessionRequests += 1
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart() {
+ throw new Error('part request was not expected')
+ },
+ }
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: `parent-${drift.label}`,
+ transport,
+ }).start()
+ ).rejects.toMatchObject({
+ name: 'DesktopUploadCapabilityError',
+ message: 'GFS resumable upload capabilities are unavailable',
+ cause: expect.objectContaining({ message: expect.stringContaining(drift.cause) }),
+ })
+ expect(sessionRequests).toBe(0)
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('resumes from sparse status and does not replay a committed part', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-resume-'))
try {
@@ -101,7 +162,7 @@ describe('desktop GFS indexed uploader', () => {
await writeFile(filePath, bytes)
const committedSha = createHash('sha256')
.update(bytes.subarray(0, 1024 * 1024))
- .digest('base64')
+ .digest('hex')
const partNumbers: number[] = []
const session = {
uploadId: '22222222-2222-4222-8222-222222222222',
@@ -120,7 +181,7 @@ describe('desktop GFS indexed uploader', () => {
const transport = {
async requestJson<T>(method: 'GET' | 'POST' | 'HEAD' | 'DELETE', url: string): Promise<T> {
if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
- return { ok: true, data: { upload: { resumableV2: { enabled: true } } } } as T
+ return enabledCapabilities() as T
if (method === 'HEAD') return {} as T
if (method === 'GET' && url.includes('/status'))
return {
@@ -188,7 +249,7 @@ describe('desktop GFS indexed uploader', () => {
const filePath = join(root, 'payload.bin')
const original = Buffer.alloc(1024, 0x31)
await writeFile(filePath, Buffer.alloc(1024, 0x42))
- const committedSha = createHash('sha256').update(original).digest('base64')
+ const committedSha = createHash('sha256').update(original).digest('hex')
const session = {
uploadId: '25252525-2525-4252-8252-252525252525',
drive: 'main',
@@ -208,7 +269,7 @@ describe('desktop GFS indexed uploader', () => {
const transport = {
async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
- return { ok: true, data: { upload: { resumableV2: { enabled: true } } } } as T
+ return enabledCapabilities() as T
if (method === 'HEAD') return {} as T
if (method === 'GET' && url.includes('/status'))
return {
@@ -275,7 +336,7 @@ describe('desktop GFS indexed uploader', () => {
const transport = {
async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
- return { ok: true, data: { upload: { resumableV2: { enabled: true } } } } as T
+ return enabledCapabilities() as T
if (method === 'HEAD') return {} as T
if (method === 'GET' && url.includes('/status'))
return { ok: true, data: { session, parts: [] } } as T
@@ -309,6 +370,499 @@ describe('desktop GFS indexed uploader', () => {
}
})
+ it('reconciles a committed part after its PUT response EOF times out without replaying bytes', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-response-loss-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ const bytes = Buffer.alloc(1024, 0x62)
+ await writeFile(filePath, bytes)
+ const committedSha = createHash('sha256').update(bytes).digest('hex')
+ const uploadId = '27272727-2727-4272-8272-272727272727'
+ const session = {
+ uploadId,
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: bytes.length,
+ partBytes: bytes.length,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ const requestOrder: string[] = []
+ const partTimeouts: number[] = []
+ let partRequests = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') {
+ requestOrder.push('head')
+ return {} as T
+ }
+ if (method === 'GET' && requestPath(url).endsWith('/status')) {
+ requestOrder.push('status')
+ return {
+ ok: true,
+ data: {
+ session: {
+ ...session,
+ state: 'uploading',
+ contiguousBytes: bytes.length,
+ committedBytes: bytes.length,
+ committedPartCount: 1,
+ },
+ parts: [
+ {
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: bytes.length,
+ sha256: committedSha,
+ },
+ ],
+ },
+ } as T
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete')) {
+ requestOrder.push('complete')
+ return {
+ ok: true,
+ data: {
+ ...session,
+ state: 'completed',
+ contiguousBytes: bytes.length,
+ committedBytes: bytes.length,
+ committedPartCount: 1,
+ },
+ } as T
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream,
+ timeoutMs: number
+ ) {
+ partRequests += 1
+ partTimeouts.push(timeoutMs)
+ requestOrder.push('part')
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* the writer durably commits these bytes before the response is lost */
+ }
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ },
+ }
+
+ const result = await new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-response-loss',
+ transport,
+ }).start()
+
+ expect(result.state).toBe('completed')
+ expect(partRequests).toBe(1)
+ expect(partTimeouts).toEqual([300_000])
+ expect(requestOrder).toEqual(['part', 'head', 'status', 'complete'])
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('retries the same part only after response-loss status proves it is not committed', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-response-missing-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ const bytes = Buffer.alloc(1024, 0x63)
+ await writeFile(filePath, bytes)
+ const uploadId = '28282828-2828-4282-8282-282828282828'
+ const session = {
+ uploadId,
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: bytes.length,
+ partBytes: bytes.length,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ const requestOrder: string[] = []
+ let partRequests = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') {
+ requestOrder.push('head')
+ return {} as T
+ }
+ if (method === 'GET' && requestPath(url).endsWith('/status')) {
+ requestOrder.push('status')
+ return {
+ ok: true,
+ data: { session: { ...session, state: 'uploading' }, parts: [] },
+ } as T
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete')) {
+ requestOrder.push('complete')
+ return {
+ ok: true,
+ data: {
+ ...session,
+ state: 'completed',
+ contiguousBytes: bytes.length,
+ committedBytes: bytes.length,
+ committedPartCount: 1,
+ },
+ } as T
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partRequests += 1
+ requestOrder.push('part')
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* drain */
+ }
+ if (partRequests === 1) {
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ }
+ return { status: 204, text: '' }
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-response-missing',
+ transport,
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+
+ expect(partRequests).toBe(2)
+ expect(requestOrder).toEqual(['part', 'head', 'status', 'part', 'complete'])
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('never adopts or replays a response-loss part whose status identity differs', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-response-drift-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ const bytes = Buffer.alloc(1024, 0x64)
+ await writeFile(filePath, bytes)
+ const checksum = createHash('sha256').update(bytes).digest('hex')
+ const driftCases = [
+ {
+ label: 'part number',
+ part: { partNumber: 1, offsetBytes: 0, lengthBytes: bytes.length, sha256: checksum },
+ },
+ {
+ label: 'offset',
+ part: { partNumber: 0, offsetBytes: 1, lengthBytes: bytes.length, sha256: checksum },
+ },
+ {
+ label: 'length',
+ part: { partNumber: 0, offsetBytes: 0, lengthBytes: bytes.length - 1, sha256: checksum },
+ },
+ {
+ label: 'checksum',
+ part: {
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: bytes.length,
+ sha256: '0'.repeat(64),
+ },
+ },
+ ]
+
+ for (const [index, drift] of driftCases.entries()) {
+ const session = {
+ uploadId: `29292929-2929-4292-8292-29292929292${index}`,
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: bytes.length,
+ partBytes: bytes.length,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let partRequests = 0
+ let completeRequests = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') return {} as T
+ if (method === 'GET' && requestPath(url).endsWith('/status'))
+ return {
+ ok: true,
+ data: {
+ session: {
+ ...session,
+ state: 'uploading',
+ contiguousBytes: bytes.length,
+ committedBytes: bytes.length,
+ committedPartCount: 1,
+ },
+ parts: [drift.part],
+ },
+ } as T
+ if (method === 'POST' && requestPath(url).endsWith('/complete')) {
+ completeRequests += 1
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partRequests += 1
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* writer committed, but the response is lost */
+ }
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: `parent-response-drift-${drift.label}`,
+ transport,
+ }).start()
+ ).rejects.toThrow('One or more upload parts could not be committed.')
+ expect(partRequests, drift.label).toBe(1)
+ expect(completeRequests, drift.label).toBe(0)
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('does not replay status-identity mismatches through the concurrency fallback', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-drift-fallback-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ const bytes = Buffer.from([0x71, 0x72, 0x73, 0x74])
+ await writeFile(filePath, bytes)
+ const uploadId = '31313131-3131-4313-8313-313131313131'
+ const session = {
+ uploadId,
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: bytes.length,
+ partBytes: 1,
+ partCount: bytes.length,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let partRequests = 0
+ let statusRequests = 0
+ let completeRequests = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities({
+ maxConcurrentPartsPerSession: 4,
+ fallbackConcurrency: 2,
+ }) as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') return {} as T
+ if (method === 'GET' && requestPath(url).endsWith('/status')) {
+ statusRequests += 1
+ return {
+ ok: true,
+ data: {
+ session: { ...session, state: 'uploading' },
+ parts: [
+ {
+ partNumber: session.partCount,
+ offsetBytes: 0,
+ lengthBytes: 1,
+ sha256: '0'.repeat(64),
+ },
+ ],
+ },
+ } as T
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete')) {
+ completeRequests += 1
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partRequests += 1
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* the response is ambiguous, so status identity controls replay safety */
+ }
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-response-drift-fallback',
+ transport,
+ }).start()
+ ).rejects.toThrow('One or more upload parts could not be committed.')
+
+ expect(partRequests).toBe(session.partCount)
+ expect(statusRequests).toBe(session.partCount)
+ expect(completeRequests).toBe(0)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('leaves an unresolved response-loss session persisted without replaying the part', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-part-outcome-unknown-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ const bytes = Buffer.alloc(1024, 0x65)
+ await writeFile(filePath, bytes)
+ const session = {
+ uploadId: '30303030-3030-4303-8303-303030303030',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: bytes.length,
+ partBytes: bytes.length,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let partRequests = 0
+ let reconcileRequests = 0
+ let completeRequests = 0
+ let persisted = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'HEAD') {
+ reconcileRequests += 1
+ throw new Error('status transport unavailable')
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete')) {
+ completeRequests += 1
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ }
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partRequests += 1
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* the result becomes ambiguous after all bytes leave Desktop */
+ }
+ const timeout = new Error('response body did not reach EOF')
+ timeout.name = 'TimeoutError'
+ throw timeout
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-outcome-unknown',
+ transport,
+ onPersist: () => {
+ persisted += 1
+ },
+ }).start()
+ ).rejects.toThrow('One or more upload parts could not be committed.')
+
+ expect(persisted).toBe(1)
+ expect(partRequests).toBe(1)
+ expect(reconcileRequests).toBe(3)
+ expect(completeRequests).toBe(0)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('refuses to complete when the source changes between parts and complete', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-source-'))
try {
@@ -331,8 +885,7 @@ describe('desktop GFS indexed uploader', () => {
}
const transport = {
async requestJson<T>(method: 'GET' | 'POST' | 'HEAD' | 'DELETE', url: string): Promise<T> {
- if (method === 'GET')
- return { ok: true, data: { upload: { resumableV2: { enabled: true } } } } as T
+ if (method === 'GET') return enabledCapabilities() as T
if (method === 'POST' && requestPath(url).endsWith('/uploads'))
return { ok: true, data: session } as T
if (method === 'POST' && requestPath(url).endsWith('/complete')) {
@@ -395,20 +948,18 @@ describe('desktop GFS indexed uploader', () => {
expiresAt: new Date(Date.now() + 60_000).toISOString(),
}
const transport = {
- async requestJson<T>(method: 'GET' | 'POST', url: string): Promise<T> {
- if (method === 'GET')
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities({
+ maxFileBytes: 200 * 1024 * 1024,
+ maxConcurrentPartsPerSession: 4,
+ fallbackConcurrency: 2,
+ }) as T
+ if (method === 'HEAD') return {} as T
+ if (method === 'GET' && requestPath(url).endsWith('/status'))
return {
ok: true,
- data: {
- upload: {
- resumableV2: {
- enabled: true,
- maxFileBytes: 200 * 1024 * 1024,
- maxConcurrentPartsPerSession: 4,
- fallbackConcurrency: 2,
- },
- },
- },
+ data: { session: { ...session, state: 'uploading' }, parts: [] },
} as T
if (method === 'POST' && requestPath(url).endsWith('/uploads')) {
completeCalls += 1
@@ -484,7 +1035,7 @@ describe('desktop GFS indexed uploader', () => {
const path = requestPath(url)
if (path.endsWith('/capabilities')) {
seen.add('capabilities')
- return { ok: true, data: { upload: { resumableV2: { enabled: true } } } } as T
+ return enabledCapabilities() as T
}
if (method === 'POST' && path.endsWith('/uploads')) {
seen.add('create')
@@ -635,15 +1186,7 @@ describe('desktop GFS indexed uploader', () => {
}
const transport = {
async requestJson<T>(method: 'GET' | 'POST', url: string): Promise<T> {
- if (method === 'GET')
- return {
- ok: true,
- data: {
- upload: {
- resumableV2: { enabled: true, maxConcurrentPartsPerSession: 1 },
- },
- },
- } as T
+ if (method === 'GET') return enabledCapabilities({ maxConcurrentPartsPerSession: 1 }) as T
if (requestPath(url).endsWith('/uploads')) return { ok: true, data: session } as T
completeRequests += 1
return { ok: true, data: { ...session, state: 'completed' } } as T
diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.ts
index 6d9c4983a..22a586986 100644
--- a/desktop-app/src/gfs/upload.ts
+++ b/desktop-app/src/gfs/upload.ts
@@ -8,6 +8,9 @@ const GFS_UPLOAD_V2_PREFERRED_PART_BYTES = 8 * 1024 * 1024
const GFS_UPLOAD_V2_MAX_PART_BYTES = 16 * 1024 * 1024
const GFS_UPLOAD_V2_DEFAULT_CONCURRENCY = 4
const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2
+const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000
+const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000
+const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
export interface DesktopUploadSession {
uploadId: string
@@ -31,6 +34,7 @@ export interface DesktopUploadPart {
partNumber: number
offsetBytes: number
lengthBytes: number
+ /** Canonical lowercase/uppercase-insensitive hex digest returned by status. */
sha256: string
}
@@ -82,6 +86,24 @@ interface UploadCapabilities {
}
}
+function parseUploadCapabilities(value: unknown): UploadCapabilities {
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
+ throw new Error('invalid GFS upload capabilities response: expected a plain JSON object')
+ }
+ const upload = (value as { upload?: unknown }).upload
+ if (!upload || typeof upload !== 'object' || Array.isArray(upload)) {
+ throw new Error('invalid GFS upload capabilities response: missing upload object')
+ }
+ const resumableV2 = (upload as { resumableV2?: unknown }).resumableV2
+ if (!resumableV2 || typeof resumableV2 !== 'object' || Array.isArray(resumableV2)) {
+ throw new Error('invalid GFS upload capabilities response: missing resumableV2 object')
+ }
+ if (typeof (resumableV2 as { enabled?: unknown }).enabled !== 'boolean') {
+ throw new Error('invalid GFS upload capabilities response: resumableV2.enabled must be boolean')
+ }
+ return value as UploadCapabilities
+}
+
export class DesktopUploadCapabilityError extends Error {
constructor(message: string, options?: { cause?: unknown }) {
super(message, options)
@@ -193,7 +215,10 @@ function retryable(status: number): boolean {
function retryableError(error: unknown): boolean {
if (error instanceof Error && error.name === 'AbortError') return false
if (error instanceof Error && error.message.startsWith('source_changed:')) return false
+ if (error instanceof Error && error.message.startsWith('resume_part_mismatch:')) return false
if (error instanceof Error && error.message.startsWith('stale_auth_epoch:')) return false
+ if (error instanceof Error && error.message.startsWith('upload_part_outcome_unknown:'))
+ return false
const status =
error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'
? Number((error as { status: number }).status)
@@ -207,7 +232,16 @@ function sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms))
}
-async function checksumPart(filePath: string, start: number, end: number): Promise<string> {
+interface DesktopUploadPartChecksum {
+ base64: string
+ hex: string
+}
+
+async function checksumPart(
+ filePath: string,
+ start: number,
+ end: number
+): Promise<DesktopUploadPartChecksum> {
const hash = createHash('sha256')
for await (const chunk of createReadStream(filePath, {
start,
@@ -216,7 +250,8 @@ async function checksumPart(filePath: string, start: number, end: number): Promi
})) {
hash.update(chunk as Buffer)
}
- return hash.digest('base64')
+ const digest = hash.digest()
+ return { base64: digest.toString('base64'), hex: digest.toString('hex') }
}
async function validateResumedParts(
@@ -224,7 +259,27 @@ async function validateResumedParts(
session: DesktopUploadSession,
parts: readonly DesktopUploadPart[]
): Promise<Set<number>> {
- const committed = new Set<number>()
+ const committed = indexCommittedParts(session, parts)
+ for (const part of committed.values()) {
+ const localChecksum = await checksumPart(
+ filePath,
+ part.offsetBytes,
+ part.offsetBytes + part.lengthBytes - 1
+ )
+ if (localChecksum.hex !== part.sha256.toLowerCase()) {
+ throw new Error(
+ `source_changed: committed part ${part.partNumber} differs from the local file`
+ )
+ }
+ }
+ return new Set(committed.keys())
+}
+
+function indexCommittedParts(
+ session: DesktopUploadSession,
+ parts: readonly DesktopUploadPart[]
+): Map<number, DesktopUploadPart> {
+ const committed = new Map<number, DesktopUploadPart>()
for (const part of parts) {
if (
!Number.isSafeInteger(part.partNumber) ||
@@ -247,17 +302,12 @@ async function validateResumedParts(
`resume_part_mismatch: committed part ${part.partNumber} was returned more than once`
)
}
- const localChecksum = await checksumPart(
- filePath,
- part.offsetBytes,
- part.offsetBytes + part.lengthBytes - 1
- )
- if (localChecksum !== part.sha256) {
+ if (!/^[0-9a-f]{64}$/i.test(part.sha256)) {
throw new Error(
- `source_changed: committed part ${part.partNumber} differs from the local file`
+ `resume_part_mismatch: committed part ${part.partNumber} has an invalid SHA-256 digest`
)
}
- committed.add(part.partNumber)
+ committed.set(part.partNumber, part)
}
return committed
}
@@ -289,6 +339,145 @@ async function assertSourceIdentity(input: DesktopUploadInput): Promise<void> {
}
}
+async function loadUploadStatus(
+ input: DesktopUploadInput,
+ uploadId: string,
+ expected: {
+ drive: string
+ expectedBytes: number
+ partBytes?: number
+ partCount?: number
+ },
+ signal?: AbortSignal,
+ timeoutMs = UPLOAD_TIMEOUT_MS
+): Promise<DesktopUploadStatus> {
+ assertAuthEpoch(input)
+ await input.transport.requestJson<Envelope<undefined>>(
+ 'HEAD',
+ joinUrl(
+ input.baseUrl,
+ uploadPath(expected.drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}`)
+ ),
+ { token: input.token, timeoutMs, signal }
+ )
+
+ let cursor: string | undefined
+ let session: DesktopUploadSession | null = null
+ const parts: DesktopUploadPart[] = []
+ for (let page = 0; page < 8; page += 1) {
+ assertAuthEpoch(input)
+ const query = new URLSearchParams({ limit: '256' })
+ if (cursor) query.set('cursor', cursor)
+ const status = unwrap<DesktopUploadStatus>(
+ await input.transport.requestJson<Envelope<DesktopUploadStatus>>(
+ 'GET',
+ joinUrl(
+ input.baseUrl,
+ uploadPath(
+ expected.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}/status`,
+ query
+ )
+ ),
+ { token: input.token, timeoutMs, signal }
+ )
+ )
+ session ??= status.session
+ if (
+ session.uploadId !== status.session.uploadId ||
+ session.uploadId !== uploadId ||
+ session.expectedBytes !== status.session.expectedBytes ||
+ session.expectedBytes !== expected.expectedBytes ||
+ session.drive !== status.session.drive ||
+ session.drive !== expected.drive ||
+ (expected.partBytes !== undefined && status.session.partBytes !== expected.partBytes) ||
+ (expected.partCount !== undefined && status.session.partCount !== expected.partCount)
+ ) {
+ throw new Error('upload status changed during reconciliation')
+ }
+ parts.push(...status.parts)
+ if (!status.nextCursor) return { session, parts, nextCursor: null }
+ cursor = status.nextCursor
+ if (page === 7) throw new Error('upload status pagination exceeded the bounded page limit')
+ }
+ throw new Error('upload status pagination did not terminate')
+}
+
+function ambiguousPartResponse(status: number | undefined): boolean {
+ // A missing response, edge timeout, or proxy/server 5xx may occur after GFSC
+ // committed the part. 425/429 are explicit admission outcomes and can retry
+ // without an extra status round-trip.
+ return status === undefined || status === 408 || status >= 500
+}
+
+function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {
+ return new Error(
+ `upload_part_outcome_unknown: could not prove whether part ${partNumber} committed; resume the persisted session`,
+ { cause }
+ )
+}
+
+async function reconcileAmbiguousPart(
+ input: DesktopUploadInput,
+ session: DesktopUploadSession,
+ partNumber: number,
+ start: number,
+ length: number,
+ checksum: DesktopUploadPartChecksum,
+ signal?: AbortSignal
+): Promise<'committed' | 'missing'> {
+ let lastError: unknown
+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {
+ let status: DesktopUploadStatus
+ try {
+ status = await loadUploadStatus(
+ input,
+ session.uploadId,
+ {
+ drive: session.drive,
+ expectedBytes: session.expectedBytes,
+ partBytes: session.partBytes,
+ partCount: session.partCount,
+ },
+ signal,
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS
+ )
+ } catch (error) {
+ if (signal?.aborted) throw signal.reason ?? error
+ lastError = error
+ if (!retryableError(error) || attempt + 1 >= GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)
+ throw partOutcomeUnknown(partNumber, error)
+ await sleep(250 * 2 ** attempt)
+ continue
+ }
+
+ const committed = indexCommittedParts(status.session, status.parts).get(partNumber)
+ if (committed) {
+ if (committed.offsetBytes !== start || committed.lengthBytes !== length) {
+ throw new Error(
+ `resume_part_mismatch: committed part ${partNumber} does not match the upload geometry`
+ )
+ }
+ if (committed.sha256.toLowerCase() !== checksum.hex) {
+ throw new Error(`source_changed: committed part ${partNumber} differs from the local file`)
+ }
+ return 'committed'
+ }
+
+ if (['aborted', 'canceling', 'failed', 'expired'].includes(status.session.state)) {
+ throw new Error(`cannot reconcile upload part in ${status.session.state}`)
+ }
+ // activePartCount is the writer's durable lease fence. Zero plus an absent
+ // committed row proves this part is safe to retry; a positive count cannot
+ // identify which concurrent part owns the lease, so poll status rather than
+ // risking an overlapping replay.
+ if (status.session.activePartCount === 0) return 'missing'
+ lastError = new Error(`writer still reports ${status.session.activePartCount} active parts`)
+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS) await sleep(250 * 2 ** attempt)
+ }
+ throw partOutcomeUnknown(partNumber, lastError)
+}
+
async function putPart(
input: DesktopUploadInput,
session: DesktopUploadSession,
@@ -327,17 +516,31 @@ async function putPart(
'upload-part-number': String(partNumber),
'upload-offset': String(start),
'upload-chunk-length': String(length),
- 'upload-checksum': `sha256 ${checksum}`,
+ 'upload-checksum': `sha256 ${checksum.base64}`,
'content-length': String(length),
},
body,
- UPLOAD_TIMEOUT_MS,
+ GFS_UPLOAD_V2_PART_TIMEOUT_MS,
signal
)
} catch (error) {
if (signal?.aborted) throw signal.reason ?? error
- if (!retryableError(error) || attempt >= 2) throw error
+ if (!retryableError(error)) throw error
onRetryableFailure?.()
+ const reconciled = await reconcileAmbiguousPart(
+ input,
+ session,
+ partNumber,
+ start,
+ length,
+ checksum,
+ signal
+ )
+ if (reconciled === 'committed') {
+ onPartSuccess?.()
+ return
+ }
+ if (attempt >= 2) throw error
await sleep(250 * 2 ** attempt)
attempt += 1
continue
@@ -350,6 +553,21 @@ async function putPart(
throw uploadPartError(response.status, response.text)
}
onRetryableFailure?.()
+ if (ambiguousPartResponse(response.status)) {
+ const reconciled = await reconcileAmbiguousPart(
+ input,
+ session,
+ partNumber,
+ start,
+ length,
+ checksum,
+ signal
+ )
+ if (reconciled === 'committed') {
+ onPartSuccess?.()
+ return
+ }
+ }
if (attempt >= 2) throw uploadPartError(response.status, response.text)
await sleep(250 * 2 ** attempt)
attempt += 1
@@ -737,8 +955,11 @@ export class DesktopGfsUploadJob {
let capabilities: UploadCapabilities
try {
this.assertAuthEpoch()
- capabilities = unwrap<UploadCapabilities>(
- await this.input.transport.requestJson<Envelope<UploadCapabilities>>(
+ // GFSC capabilities are a plain JSON document. Only upload lifecycle
+ // receipts use the { ok, data, error } envelope; accepting an envelope
+ // here would hide drift between Desktop and the writer's public contract.
+ capabilities = parseUploadCapabilities(
+ await this.input.transport.requestJson<UploadCapabilities>(
'GET',
joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/capabilities')),
{
@@ -764,58 +985,13 @@ export class DesktopGfsUploadJob {
this.input.fallbackConcurrency = resumable.fallbackConcurrency
const resumeId = this.session?.uploadId ?? this.input.resumeUploadId
if (resumeId) {
- this.assertAuthEpoch()
- await this.input.transport.requestJson<Envelope<undefined>>(
- 'HEAD',
- joinUrl(
- this.input.baseUrl,
- uploadPath(drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(resumeId)}`)
- ),
- {
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.abortController.signal,
- }
+ const status = await loadUploadStatus(
+ this.input,
+ resumeId,
+ { drive, expectedBytes: file.size },
+ this.abortController.signal
)
- let cursor: string | undefined
- let session: DesktopUploadSession | null = null
- const parts: DesktopUploadPart[] = []
- for (let page = 0; page < 8; page += 1) {
- this.assertAuthEpoch()
- const query = new URLSearchParams({ limit: '256' })
- if (cursor) query.set('cursor', cursor)
- const status = unwrap<DesktopUploadStatus>(
- await this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(
- 'GET',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(resumeId)}/status`,
- query
- )
- ),
- {
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.abortController.signal,
- }
- )
- )
- session ??= status.session
- if (
- session.uploadId !== status.session.uploadId ||
- session.expectedBytes !== status.session.expectedBytes ||
- status.session.drive !== drive
- )
- throw new Error('upload status changed during resume')
- parts.push(...status.parts)
- if (!status.nextCursor) break
- cursor = status.nextCursor
- if (page === 7) throw new Error('upload status pagination exceeded the bounded page limit')
- }
- if (!session || session.expectedBytes !== file.size || session.drive !== drive)
- throw new Error('selected file size differs from the upload session')
+ const { session, parts } = status
const committed =
session.state === 'completed'
? new Set<number>()
@@ -886,7 +1062,15 @@ export class DesktopGfsUploadJob {
if (prepared.session.state === 'paused') return prepared.session
try {
const result = await runParts(
- this.input,
+ {
+ ...this.input,
+ onProgress: (uploadedBytes, totalBytes) => {
+ this.uploadedBytes = uploadedBytes
+ this.totalBytes = totalBytes
+ this.emit()
+ this.input.onProgress?.(uploadedBytes, totalBytes)
+ },
+ },
prepared.session,
prepared.committed,
this.abortController.signal,
diff --git a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
index 9a30732e3..3f034547c 100644
--- a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
+++ b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
@@ -1,4 +1,4 @@
-import type { Download, Locator, Page } from '@playwright/test'
+import type { Locator, Page } from '@playwright/test'
import { expect } from '@playwright/test'
import { createHash } from 'node:crypto'
import {
@@ -21,16 +21,186 @@ import { openResourcesNavItem } from './navigationHelpers'
test.describe.configure({ mode: 'serial' })
-async function hashDownload(download: Download): Promise<{ bytes: number; sha256: string }> {
- const stream = await download.createReadStream()
- if (!stream) throw new Error(`download stream unavailable for ${download.suggestedFilename()}`)
+async function selectFileThroughVisibleAction(
+ page: Page,
+ trigger: Locator,
+ filePath: string
+): Promise<void> {
+ const fileChooserPromise = page.waitForEvent('filechooser')
+ await trigger.click()
+ const fileChooser = await fileChooserPromise
+ await fileChooser.setFiles(filePath)
+}
+
+async function observeProgressOrCompletion(
+ page: Page,
+ progress: Locator,
+ completedText: string,
+ timeout = 900_000
+): Promise<void> {
+ const values = new Set<string>()
+ const completed = page.getByText(completedText, { exact: true })
+ const errorToast = page.getByRole('alert').filter({ hasText: /file|gfs|upload/i })
+ let outcome: 'waiting' | 'progress' | 'error' | `completed:${number}` = 'waiting'
+ await expect
+ .poll(
+ async () => {
+ if ((await progress.count()) > 0) {
+ const value = await progress.getAttribute('aria-valuenow', { timeout: 250 })
+ if (value !== null) values.add(value)
+ }
+ outcome = values.size >= 2 ? 'progress' : outcome
+ if (outcome === 'waiting' && (await completed.isVisible().catch(() => false))) {
+ outcome = `completed:${values.size}`
+ }
+ if (outcome === 'waiting' && (await errorToast.isVisible().catch(() => false))) {
+ outcome = 'error'
+ }
+ return outcome !== 'waiting'
+ },
+ { timeout, intervals: [50, 250, 1_000] }
+ )
+ .toBeTruthy()
+
+ if (outcome === 'progress') return
+ if (outcome === 'error') {
+ throw new Error(
+ `Desktop rejected the selected local file before two progress values: ${(await errorToast.textContent())?.trim() || 'Unknown Desktop upload error'}`
+ )
+ }
+ if (outcome.startsWith('completed:')) return
+ throw new Error(`Desktop upload progress observation ended unexpectedly (${outcome})`)
+}
+
+function fileResourceControls(browser: Locator, resourceName: string) {
+ return {
+ name: browser.getByRole('button', { name: resourceName, exact: true }),
+ options: browser.getByRole('button', {
+ name: `Options for ${resourceName}`,
+ exact: true,
+ }),
+ download: browser.getByRole('button', {
+ name: `Download ${resourceName}`,
+ exact: true,
+ }),
+ }
+}
+
+async function returnToFolder(browser: Locator, folderName: string): Promise<void> {
+ const location = browser.getByRole('navigation', { name: 'File location' })
+ const sharedRoot = location.getByRole('button', { name: 'Shared with me', exact: true })
+ await expect(sharedRoot).toBeEnabled()
+ await sharedRoot.click()
+ const folder = browser.getByRole('button', { name: folderName, exact: true })
+ await expect(folder).toBeVisible({ timeout: 60_000 })
+ await folder.click()
+ await expect(location.getByRole('button', { name: folderName, exact: true })).toBeDisabled({
+ timeout: 30_000,
+ })
+}
+
+async function installGfsDownloadCapture(page: Page): Promise<void> {
+ await page.evaluate(() => {
+ type CapturedGfsDownload = {
+ filename: string
+ blob: Blob
+ }
+ const win = window as Window & {
+ __clerumE2eGfsDownloads?: CapturedGfsDownload[]
+ __clerumE2eGfsDownloadCaptureInstalled?: boolean
+ }
+ if (win.__clerumE2eGfsDownloadCaptureInstalled) return
+ win.__clerumE2eGfsDownloadCaptureInstalled = true
+ win.__clerumE2eGfsDownloads = []
+
+ // Observe the Blob produced from the real preload/IPC response while
+ // preserving both production APIs. This does not replace or mock GFS.
+ const blobByUrl = new Map<string, Blob>()
+ const originalCreateObjectURL = URL.createObjectURL.bind(URL)
+ URL.createObjectURL = (object: Blob | MediaSource) => {
+ const url = originalCreateObjectURL(object)
+ if (object instanceof Blob) blobByUrl.set(url, object)
+ return url
+ }
+
+ const originalAnchorClick = HTMLAnchorElement.prototype.click
+ HTMLAnchorElement.prototype.click = function (this: HTMLAnchorElement) {
+ const filename = this.download
+ const blob = blobByUrl.get(this.href)
+ if (filename && blob) win.__clerumE2eGfsDownloads?.push({ filename, blob })
+ return originalAnchorClick.call(this)
+ }
+ })
+}
+
+async function hashCapturedGfsDownload(
+ page: Page,
+ filename: string,
+ bytes: number
+): Promise<string> {
const hash = createHash('sha256')
- let bytes = 0
- for await (const chunk of stream) {
- bytes += chunk.length
- hash.update(chunk)
+ const chunkBytes = 8 * 1024 * 1024
+ for (let offset = 0; offset < bytes; offset += chunkBytes) {
+ const encoded = await page.evaluate(
+ async ({ filenameToFind, start, end }) => {
+ const downloads =
+ (
+ window as Window & {
+ __clerumE2eGfsDownloads?: Array<{ filename: string; blob: Blob }>
+ }
+ ).__clerumE2eGfsDownloads ?? []
+ const download = downloads.find(item => item.filename === filenameToFind)
+ if (!download) throw new Error(`captured GFS download not found: ${filenameToFind}`)
+ const chunk = new Uint8Array(await download.blob.slice(start, end).arrayBuffer())
+ let binary = ''
+ for (let index = 0; index < chunk.length; index += 0x8000) {
+ binary += String.fromCharCode(...chunk.subarray(index, index + 0x8000))
+ }
+ return btoa(binary)
+ },
+ { filenameToFind: filename, start: offset, end: Math.min(bytes, offset + chunkBytes) }
+ )
+ hash.update(Buffer.from(encoded, 'base64'))
}
- return { bytes, sha256: hash.digest('hex') }
+ return hash.digest('hex')
+}
+
+async function expectGfsDownload(
+ page: Page,
+ button: Locator,
+ filename: string,
+ expected: { bytes: number; sha256: string }
+): Promise<void> {
+ await installGfsDownloadCapture(page)
+ await expect(button).toBeVisible()
+ await button.click()
+ await expect(page.getByText(`Downloaded ${filename}`, { exact: true })).toBeVisible({
+ timeout: 300_000,
+ })
+ await expect
+ .poll(
+ () =>
+ page.evaluate(filenameToFind => {
+ const downloads =
+ (
+ window as Window & {
+ __clerumE2eGfsDownloads?: Array<{
+ filename: string
+ blob: Blob
+ }>
+ }
+ ).__clerumE2eGfsDownloads ?? []
+ const download = downloads.find(item => item.filename === filenameToFind)
+ return download ? { filename: download.filename, bytes: download.blob.size } : null
+ }, filename),
+ {
+ timeout: 300_000,
+ intervals: [100, 500, 1_000],
+ message: `Desktop should expose the real GFS IPC bytes for ${filename}`,
+ }
+ )
+ .toEqual({ filename, bytes: expected.bytes })
+ expect(await hashCapturedGfsDownload(page, filename, expected.bytes)).toBe(expected.sha256)
}
async function openFolder(
@@ -45,15 +215,23 @@ async function openFolder(
await expect(page.getByRole('heading', { name: 'Files', exact: true })).toBeVisible({
timeout: 30_000,
})
- const shared = page.getByRole('region', { name: 'GFS resources shared with you' })
- await expect(shared.getByRole('button', { name: folderName, exact: true })).toBeVisible({
+ const browser = page.getByRole('region', { name: 'Global File System browser' })
+ await expect(browser.getByRole('button', { name: folderName, exact: true })).toBeVisible({
timeout: 60_000,
})
- await shared.getByRole('button', { name: folderName, exact: true }).click()
- const browser = page.getByRole('region', { name: 'Global File System browser' })
+ await browser.getByRole('button', { name: folderName, exact: true }).click()
await expect(browser).toBeVisible({ timeout: 30_000 })
- await expect(browser.getByRole('button', { name: `Options for ${folderName}` })).toBeVisible()
- await browser.getByRole('button', { name: `Options for ${folderName}` }).click()
+ await expect(
+ browser
+ .getByRole('navigation', { name: 'File location' })
+ .getByRole('button', { name: folderName, exact: true })
+ ).toBeDisabled({ timeout: 30_000 })
+ const currentFolderOptions = browser.getByRole('button', {
+ name: `Options for ${folderName}`,
+ exact: true,
+ })
+ await expect(currentFolderOptions).toBeVisible()
+ await currentFolderOptions.click()
await browser.getByRole('menuitem', { name: 'Manage', exact: true }).click()
const manageDialog = page.getByRole('dialog', {
name: `Manage folder ${folderName}`,
@@ -63,20 +241,6 @@ async function openFolder(
return { browser, manageDialog }
}
-async function observeTwoProgressValues(progress: Locator, timeout = 900_000): Promise<void> {
- const values = new Set<string>()
- await expect
- .poll(
- async () => {
- const value = await progress.getAttribute('aria-valuenow')
- if (value !== null) values.add(value)
- return values.size
- },
- { timeout, intervals: [250, 1_000] }
- )
- .toBeGreaterThanOrEqual(2)
-}
-
async function grantFolderToE2eUser(resourceId: string): Promise<void> {
seedGfsGrant({
resourceId,
@@ -95,24 +259,22 @@ async function exerciseCreate(page: Page, byteLength: number): Promise<void> {
const source = await createDiskUploadFixture(byteLength, '.parquet', fixtureName)
try {
const { browser, manageDialog } = await openFolder(page, fixture.name)
- const uploadResponsePromise = page.waitForResponse(
- response => response.request().method() === 'GET' && response.url().includes('/external/gfs')
+ await selectFileThroughVisibleAction(
+ page,
+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),
+ source.filePath
)
- await manageDialog.getByRole('button', { name: 'Upload file', exact: true }).click()
- await page.getByLabel('Upload file', { exact: true }).setInputFiles(source.filePath)
- await uploadResponsePromise.catch(() => undefined)
const progress = browser.getByRole('progressbar', {
name: `GFS upload progress for ${source.fileName}`,
})
- await expect(progress).toBeVisible({ timeout: 30_000 })
- await observeTwoProgressValues(progress)
+ await observeProgressOrCompletion(page, progress, `Uploaded ${source.fileName}`)
await expect(page.getByText(`Uploaded ${source.fileName}`, { exact: true })).toBeVisible({
timeout: 1_200_000,
})
await page.getByRole('button', { name: 'Close manage dialog' }).click()
- const row = browser.getByRole('listitem').filter({ hasText: source.fileName })
- await expect(row).toBeVisible({ timeout: 60_000 })
- await expect(row).toContainText('200.0 MB')
+ const resource = fileResourceControls(browser, source.fileName)
+ await expect(resource.name).toBeVisible({ timeout: 60_000 })
+ await expect(browser.getByText(/^200\.0 MB · v\d+$/)).toBeVisible()
await expect
.poll(
() =>
@@ -123,9 +285,7 @@ async function exerciseCreate(page: Page, byteLength: number): Promise<void> {
{ timeout: 60_000, intervals: [250, 1_000] }
)
.toMatchObject({ kind: 'file', bytes: byteLength, deleted: false })
- const downloadPromise = page.waitForEvent('download')
- await row.getByRole('button', { name: `Download ${source.fileName}` }).click()
- expect(await hashDownload(await downloadPromise)).toEqual({
+ await expectGfsDownload(page, resource.download, source.fileName, {
bytes: byteLength,
sha256: source.sha256,
})
@@ -149,30 +309,29 @@ async function exerciseReplace(page: Page, byteLength: number): Promise<void> {
})
expect(before).toMatchObject({ kind: 'file', deleted: false })
await page.getByRole('button', { name: 'Close manage dialog' }).click()
- const fileRow = browser.getByRole('listitem').filter({ hasText: fixture.fileName })
- await expect(fileRow).toBeVisible({ timeout: 30_000 })
- await fileRow.getByRole('button', { name: `Options for ${fixture.fileName}` }).click()
+ const resource = fileResourceControls(browser, fixture.fileName)
+ await expect(resource.name).toBeVisible({ timeout: 30_000 })
+ await resource.options.click()
await browser.getByRole('menuitem', { name: 'Manage', exact: true }).click()
const manageFile = page.getByRole('dialog', {
name: `Manage file ${fixture.fileName}`,
exact: true,
})
await expect(manageFile).toBeVisible()
- const uploadResponsePromise = page.waitForResponse(
- response => response.request().method() === 'GET' && response.url().includes('/external/gfs')
+ await selectFileThroughVisibleAction(
+ page,
+ manageFile.getByRole('button').filter({ hasText: /^Replace file$/ }),
+ source.filePath
)
- await manageFile.getByRole('button', { name: 'Replace file', exact: true }).click()
- await page.getByLabel('Replace file', { exact: true }).setInputFiles(source.filePath)
- await uploadResponsePromise.catch(() => undefined)
const progress = browser.getByRole('progressbar', {
- name: `GFS upload progress for ${source.fileName}`,
+ name: `GFS upload progress for ${fixture.fileName}`,
})
- await expect(progress).toBeVisible({ timeout: 30_000 })
- await observeTwoProgressValues(progress)
+ await observeProgressOrCompletion(page, progress, `Replaced ${fixture.fileName}`)
await expect(page.getByText(`Replaced ${fixture.fileName}`, { exact: true })).toBeVisible({
timeout: 1_200_000,
})
await page.getByRole('button', { name: 'Close manage dialog' }).click()
+ await returnToFolder(browser, fixture.name)
await expect
.poll(
() =>
@@ -188,10 +347,9 @@ async function exerciseReplace(page: Page, byteLength: number): Promise<void> {
version: (before?.version ?? 0) + 1,
deleted: false,
})
- await expect(fileRow).toContainText('200.0 MB')
- const downloadPromise = page.waitForEvent('download')
- await fileRow.getByRole('button', { name: `Download ${fixture.fileName}` }).click()
- expect(await hashDownload(await downloadPromise)).toEqual({
+ await expect(resource.name).toBeVisible({ timeout: 60_000 })
+ await expect(browser.getByText(/^200\.0 MB · v\d+$/)).toBeVisible()
+ await expectGfsDownload(page, resource.download, fixture.fileName, {
bytes: byteLength,
sha256: source.sha256,
})