Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): close resumable upload review gaps

Alfredo Lopez5bac17aa9c52
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.ts
index 0ab7bd6af..8a9e4aea4 100644
--- a/control-api/src/routes/external/gfs.ts
+++ b/control-api/src/routes/external/gfs.ts
@@ -41,6 +41,26 @@ import { GFS_DEFAULT_DRIVE, parseRequestedGfsScopes } from '../gfs/token.js'
// rejected here, not forwarded to gfsc as an unbounded request.
const MAX_GFS_URI_LEN = 2048
const ACCESSIBLE_RESOURCE_DEFAULT_LIMIT = 100
+const GFS_UPLOAD_RESPONSE_HEADERS = [
+ 'content-type',
+ 'content-length',
+ 'cache-control',
+ 'location',
+ 'upload-offset',
+ 'upload-length',
+ 'upload-part-bytes',
+ 'upload-part-count',
+ 'upload-active-parts',
+ 'upload-state',
+ 'upload-expires',
+ 'upload-part-number',
+ 'upload-part-offset',
+ 'upload-part-length',
+ 'upload-checksum',
+ 'retry-after',
+ 'x-ratelimit-limit',
+ 'x-ratelimit-remaining',
+] as const
type ExternalGfsRequest = ExternalAuthedRequest & {
gfsSubjectKeys?: string[]
@@ -925,31 +945,15 @@ async function proxyUploadToGfsc(
'gfs external upload upstream error'
)
res.status(upstream.status)
- const contentType = upstream.headers.get('content-type')
- if (contentType) res.setHeader('content-type', contentType)
+ for (const name of GFS_UPLOAD_RESPONSE_HEADERS) {
+ const value = upstream.headers.get(name)
+ if (value) res.setHeader(name, value)
+ }
res.send(body)
return
}
res.status(upstream.status)
- for (const name of [
- 'content-type',
- 'content-length',
- 'cache-control',
- 'upload-offset',
- 'upload-length',
- 'upload-part-bytes',
- 'upload-part-count',
- 'upload-active-parts',
- 'upload-state',
- 'upload-expires',
- 'upload-part-number',
- 'upload-part-offset',
- 'upload-part-length',
- 'upload-checksum',
- 'retry-after',
- 'x-ratelimit-limit',
- 'x-ratelimit-remaining',
- ]) {
+ for (const name of GFS_UPLOAD_RESPONSE_HEADERS) {
const value = upstream.headers.get(name)
if (value) res.setHeader(name, value)
}
diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.ts
index 895fa32dc..adc573f95 100644
--- a/control-api/src/routes/gfs/proxy.ts
+++ b/control-api/src/routes/gfs/proxy.ts
@@ -11,7 +11,24 @@ import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/
import { rootLogger } from '../../observability/logger.js'
const DEFAULT_DRIVE = 'main'
-const PASSTHROUGH_RESPONSE_HEADERS = ['content-type', 'content-disposition', 'content-length']
+const PASSTHROUGH_RESPONSE_HEADERS = [
+ 'content-type',
+ 'content-disposition',
+ 'content-length',
+ 'location',
+ 'upload-offset',
+ 'upload-length',
+ 'upload-part-bytes',
+ 'upload-part-count',
+ 'upload-active-parts',
+ 'upload-state',
+ 'upload-expires',
+ 'upload-part-number',
+ 'upload-part-offset',
+ 'upload-part-length',
+ 'upload-checksum',
+ 'retry-after',
+]
const UPLOAD_PART_PATH = /^\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+(?:\?|$)/i
const UPLOAD_PATH = /^\/v1\/(?:capabilities|uploads)(?:\/|\?|$)/i
const STREAM_REQUEST_HEADERS = [
@@ -137,6 +154,12 @@ export function registerGfsProxyRoute(router: Router): void {
res.status(upstreamRes.status)
const contentType = upstreamRes.headers.get('content-type')
if (contentType) res.setHeader('content-type', contentType)
+ const retryAfter = upstreamRes.headers.get('retry-after')
+ if (retryAfter) res.setHeader('retry-after', retryAfter)
+ for (const header of PASSTHROUGH_RESPONSE_HEADERS) {
+ const value = upstreamRes.headers.get(header)
+ if (value) res.setHeader(header, value)
+ }
res.send(errorBody)
return
}
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index bc9309875..d5d8ce30c 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -33,6 +33,8 @@ export function currentWindowStartMs(nowMs = Date.now()): number {
return Math.floor(nowMs / WINDOW_MS) * WINDOW_MS
}
+type RateLimitQuery = (text: string, values?: unknown[]) => Promise<{ rows: unknown[] }>
+
/**
* Atomically increment the (bucketKey, windowStart) counter, return the new
* count plus whether it's still under `maxPerMinute`.
@@ -47,6 +49,28 @@ export async function checkAndIncrement(
maxPerMinute: number,
nowMs = Date.now(),
cost = 1
+): Promise<RateLimitCheck> {
+ return checkAndIncrementWithQuery(
+ (text, values) => pool.query(text, values),
+ bucketKey,
+ maxPerMinute,
+ nowMs,
+ cost
+ )
+}
+
+/**
+ * Same atomic limiter operation against an explicitly supplied query
+ * function. Production callers use the process-wide pool above; the narrow
+ * seam lets the real-Postgres integration suite exercise this exact SQL
+ * against an isolated database without changing runtime ownership.
+ */
+export async function checkAndIncrementWithQuery(
+ query: RateLimitQuery,
+ bucketKey: string,
+ maxPerMinute: number,
+ nowMs = Date.now(),
+ cost = 1
): Promise<RateLimitCheck> {
if (!Number.isSafeInteger(cost) || cost < 1) throw new Error('rate limit cost must be positive')
const windowStartMs = currentWindowStartMs(nowMs)
@@ -54,7 +78,7 @@ export async function checkAndIncrement(
let result: { rows: unknown[] } | null | undefined
try {
- result = await pool.query(
+ result = await query(
`INSERT INTO rate_limit_buckets (bucket_key, window_start_ms, count)
VALUES ($1, $2, $3)
ON CONFLICT (bucket_key, window_start_ms) DO UPDATE
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
index 60d84dc67..06771f2f9 100644
--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
@@ -1,12 +1,12 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { createHash, randomBytes, randomUUID } from 'node:crypto'
-import { mkdtemp, rm } from 'node:fs/promises'
+import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Readable } from 'node:stream'
import { Pool } from 'pg'
import type { GfsUploadConfig } from '../../gfs-controller/src/config.js'
-import { PgTransactor } from '../../gfs-controller/src/db/writeStore.js'
+import { CommitOutcomeUnknownError, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'
import type {
TransactionOptions,
Transactor,
@@ -68,6 +68,32 @@ class HoldBeforeCommitTransactor implements Transactor {
}
}
+/**
+ * Simulates a network/process failure after PostgreSQL has committed. The
+ * upload service must reconcile the durable row instead of replaying a
+ * state-changing operation blindly.
+ */
+class ThrowAfterCommitTransactor implements Transactor {
+ private transactionCount = 0
+
+ constructor(
+ private readonly delegate: Transactor,
+ private readonly failOnTransaction: number
+ ) {}
+
+ async transaction<T>(
+ fn: (client: TxClient) => Promise<T>,
+ options?: TransactionOptions
+ ): Promise<T> {
+ this.transactionCount += 1
+ const result = await this.delegate.transaction(fn, options)
+ if (this.transactionCount === this.failOnTransaction) {
+ throw new CommitOutcomeUnknownError(new Error('synthetic post-commit response loss'))
+ }
+ return result
+ }
+}
+
const CONFIG: GfsUploadConfig = {
productMaxFileBytes: 209715200,
protocolMaxFileBytes: 1073741824,
@@ -222,6 +248,241 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(rows.rows).toEqual(expect.arrayContaining([{ state: 'committed', count: 2 }]))
})
+ it('serializes concurrent indexed part reservations without losing either commit', async () => {
+ const bytes = [
+ Buffer.alloc(CONFIG.preferredPartBytes, 0x31),
+ Buffer.alloc(CONFIG.preferredPartBytes, 0x32),
+ ]
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '7'.repeat(32),
+ name: 'concurrent-parts.bin',
+ sizeBytes: bytes.length * CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ const results = await Promise.all(
+ bytes.map((value, partNumber) =>
+ uploads.putPart(
+ created.session.uploadId,
+ partGeometry(
+ { expectedBytes: value.length * bytes.length, partBytes: CONFIG.preferredPartBytes },
+ partNumber
+ ),
+ checksum(value),
+ Readable.from([value]) as never,
+ principal
+ )
+ )
+ )
+ expect(results.map(result => result.part.state)).toEqual(['committed', 'committed'])
+ const count = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,
+ [created.session.uploadId]
+ )
+ expect(count.rows[0]?.count).toBe(2)
+ })
+
+ it('completes non-zero parts and replays complete idempotently', async () => {
+ const bytes0 = Buffer.alloc(CONFIG.preferredPartBytes, 0x61)
+ const bytes1 = Buffer.alloc(CONFIG.preferredPartBytes, 0x62)
+ const uploadsWithFinalizer = new GfsUploadSessionService({
+ db: pool,
+ tx: new PgTransactor(pool),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ finalize: async session => ({
+ resourceId: session.uploadId,
+ version: 1,
+ sha256: checksum(Buffer.concat([bytes0, bytes1])),
+ }),
+ })
+ const created = await uploadsWithFinalizer.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: 'f'.repeat(32),
+ name: 'complete-non-zero.bin',
+ sizeBytes: bytes0.length + bytes1.length,
+ idempotencyKey: randomUUID(),
+ })
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ for (const [partNumber, bytes] of [bytes0, bytes1].entries()) {
+ await uploadsWithFinalizer.putPart(
+ created.session.uploadId,
+ partGeometry(
+ { expectedBytes: bytes0.length + bytes1.length, partBytes: CONFIG.preferredPartBytes },
+ partNumber
+ ),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ principal
+ )
+ }
+
+ const completed = await uploadsWithFinalizer.complete(created.session.uploadId, principal)
+ expect(completed).toMatchObject({
+ uploadId: created.session.uploadId,
+ state: 'completed',
+ committedBytes: bytes0.length + bytes1.length,
+ resultVersion: 1,
+ })
+ await expect(
+ uploadsWithFinalizer.complete(created.session.uploadId, principal)
+ ).resolves.toEqual(completed)
+ })
+
+ it('reconciles an ambiguous committed part response without duplicating the part', async () => {
+ const base = new GfsUploadSessionService({
+ db: pool,
+ tx: new PgTransactor(pool),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ })
+ const created = await base.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '1'.repeat(32),
+ name: 'ambiguous-part.bin',
+ sizeBytes: CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ const uploadsWithUnknownCommit = new GfsUploadSessionService({
+ db: pool,
+ // The reservation transaction commits first; the second transaction is
+ // the indexed-part commit whose response is intentionally lost.
+ tx: new ThrowAfterCommitTransactor(new PgTransactor(pool), 2),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ })
+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x73)
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ const result = await uploadsWithUnknownCommit.putPart(
+ created.session.uploadId,
+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ principal
+ )
+ expect(result.part.state).toBe('committed')
+ const count = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,
+ [created.session.uploadId]
+ )
+ expect(count.rows[0]?.count).toBe(1)
+ })
+
+ it('reconciles an ambiguous complete response from the committed session row', async () => {
+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x84)
+ const base = new GfsUploadSessionService({
+ db: pool,
+ tx: new PgTransactor(pool),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ })
+ const created = await base.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '2'.repeat(32),
+ name: 'ambiguous-complete.bin',
+ sizeBytes: bytes.length,
+ idempotencyKey: randomUUID(),
+ })
+ const principal = { drive, ownerSubject: owner, primarySubject: owner }
+ await base.putPart(
+ created.session.uploadId,
+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ principal
+ )
+ const uploadsWithFinalizer = new GfsUploadSessionService({
+ db: pool,
+ tx: new ThrowAfterCommitTransactor(new PgTransactor(pool), 2),
+ blobs: { availableBytes: async () => 10n ** 12n } as never,
+ storageMountPath: tempRoot,
+ config: CONFIG,
+ finalize: async session => ({
+ resourceId: session.uploadId,
+ version: 2,
+ sha256: checksum(bytes),
+ }),
+ })
+ await expect(
+ uploadsWithFinalizer.complete(created.session.uploadId, principal)
+ ).resolves.toMatchObject({
+ state: 'completed',
+ resultVersion: 2,
+ })
+ })
+
+ it('fences and cleans a stale reserved lease while its row lock is held', async () => {
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '3'.repeat(32),
+ name: 'stale-reserved.bin',
+ sizeBytes: CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ const partsDir = join(tempRoot, '.uploads', created.session.uploadId, 'parts')
+ await mkdir(partsDir, { recursive: true })
+ const staleSuffix = `part.tmp-${randomUUID()}`
+ const stagingPath = join(partsDir, `0.${staleSuffix}`)
+ const finalPath = join(partsDir, '0.part')
+ await writeFile(stagingPath, Buffer.from('stale-temp'))
+ await writeFile(finalPath, Buffer.from('stale-final'))
+ await pool.query(
+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = 1 WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ await pool.query(
+ `INSERT INTO gfs_upload_parts
+ (upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch, lease_started_at)
+ VALUES ($1, 0, 0, $2, $3, 'reserved', $4, 1, now() - interval '1 hour')`,
+ [
+ created.session.uploadId,
+ CONFIG.preferredPartBytes,
+ checksum(Buffer.from('stale')),
+ stagingPath,
+ ]
+ )
+
+ const result = await uploads.reconcile()
+
+ expect(result.staleParts).toBeGreaterThanOrEqual(1)
+ expect(
+ await pool.query('SELECT state FROM gfs_upload_parts WHERE upload_id = $1', [
+ created.session.uploadId,
+ ])
+ ).toMatchObject({
+ rows: [{ state: 'failed' }],
+ })
+ expect(
+ (
+ await pool.query('SELECT active_part_count FROM gfs_upload_sessions WHERE upload_id = $1', [
+ created.session.uploadId,
+ ])
+ ).rows[0]?.active_part_count
+ ).toBe(0)
+ await expect(stat(stagingPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(stat(finalPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ })
+
it('does not disclose a session to another owner and makes cancel terminal', async () => {
const created = await uploads.create({
drive,
@@ -323,6 +584,63 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(parts.rows[0]?.count).toBe(0)
})
+ it('reconciles expired sessions and detects a committed part whose file is missing', async () => {
+ const expired = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '9'.repeat(32),
+ name: 'expired-reconcile.bin',
+ sizeBytes: CONFIG.preferredPartBytes,
+ idempotencyKey: randomUUID(),
+ })
+ await pool.query(
+ `UPDATE gfs_upload_sessions SET expires_at = now() - interval '1 second' WHERE upload_id = $1`,
+ [expired.session.uploadId]
+ )
+
+ const expiredResult = await uploads.reconcile()
+ expect(expiredResult.expiredSessions).toBeGreaterThanOrEqual(1)
+ await expect(
+ pool.query(`SELECT state FROM gfs_upload_sessions WHERE upload_id = $1`, [
+ expired.session.uploadId,
+ ])
+ ).resolves.toMatchObject({ rows: [{ state: 'expired' }] })
+
+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x5a)
+ const committed = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: '8'.repeat(32),
+ name: 'missing-reconcile.bin',
+ sizeBytes: bytes.length,
+ idempotencyKey: randomUUID(),
+ })
+ await uploads.putPart(
+ committed.session.uploadId,
+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),
+ checksum(bytes),
+ Readable.from([bytes]) as never,
+ { drive, ownerSubject: owner, primarySubject: owner }
+ )
+ const partRow = await pool.query<{ staging_path: string }>(
+ `SELECT staging_path FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,
+ [committed.session.uploadId]
+ )
+ await rm(String(partRow.rows[0]?.staging_path), { force: true })
+
+ const corruptResult = await uploads.reconcile()
+ expect(corruptResult).toMatchObject({ staleParts: 0 })
+ const corruptRow = await pool.query<{ state: string; failure_code: string }>(
+ `SELECT state, failure_code FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [committed.session.uploadId]
+ )
+ expect(corruptRow.rows[0]).toEqual({ state: 'failed', failure_code: 'corrupt_part_missing' })
+ })
+
it('rejects cancel during finalization and leaves one completed receipt', async () => {
let finalizerStarted!: () => void
let releaseFinalizer!: () => void
diff --git a/control-api/test/routes.gfsProxy.test.ts b/control-api/test/routes.gfsProxy.test.ts
index ae76ecfab..cb8053879 100644
--- a/control-api/test/routes.gfsProxy.test.ts
+++ b/control-api/test/routes.gfsProxy.test.ts
@@ -199,4 +199,53 @@ describe('/api/v1/gfs/proxy', () => {
expect(res.status).toBe(413)
expect(res.body).toEqual({ ok: false, error: { code: 'payload_too_large' } })
})
+
+ it('preserves upload-length and Retry-After headers for lifecycle clients', async () => {
+ const fetchMock = vi.fn(
+ async () =>
+ new Response('{"error":"quota_exceeded"}', {
+ status: 429,
+ headers: {
+ 'content-type': 'application/json',
+ 'retry-after': '3',
+ 'upload-length': '209715200',
+ },
+ })
+ )
+ vi.stubGlobal('fetch', fetchMock)
+
+ const app = await buildApp()
+ const res = await request(app).post('/api/v1/gfs/proxy/v1/uploads').send({})
+
+ expect(res.status).toBe(429)
+ expect(res.headers['retry-after']).toBe('3')
+ expect(res.headers['upload-length']).toBe('209715200')
+ })
+
+ it('preserves the bounded upload response headers on successful lifecycle responses', async () => {
+ const fetchMock = vi.fn(
+ async () =>
+ new Response(null, {
+ status: 204,
+ headers: {
+ location: '/v1/uploads/01234567-89ab-cdef-0123-456789abcdef',
+ 'upload-offset': '8388608',
+ 'upload-active-parts': '1',
+ 'upload-state': 'uploading',
+ },
+ })
+ )
+ vi.stubGlobal('fetch', fetchMock)
+
+ const app = await buildApp()
+ const res = await request(app).head(
+ '/api/v1/gfs/proxy/v1/uploads/01234567-89ab-cdef-0123-456789abcdef'
+ )
+
+ expect(res.status).toBe(204)
+ expect(res.headers.location).toContain('/v1/uploads/')
+ expect(res.headers['upload-offset']).toBe('8388608')
+ expect(res.headers['upload-active-parts']).toBe('1')
+ expect(res.headers['upload-state']).toBe('uploading')
+ })
})
diff --git a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts
new file mode 100644
index 000000000..2879879ce
--- /dev/null
+++ b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts
@@ -0,0 +1,100 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { randomBytes } from 'node:crypto'
+import { Pool } from 'pg'
+import { initDb } from '../src/db.js'
+import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+describeRealPostgres('rate limiter atomicity on real PostgreSQL', () => {
+ const database = `rate_limiter_${randomBytes(6).toString('hex')}`
+ const bucketKey = `real-pg:${randomBytes(8).toString('hex')}`
+ const connectionString = databaseUrl(
+ adminUrl ?? 'postgresql://[email protected]/postgres',
+ database
+ )
+ const windowStartMs = Math.floor(Date.now() / 60_000) * 60_000
+ let adminPool: Pool
+ let pool: Pool
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ pool = new Pool({ connectionString })
+ await initDb({ connect: () => pool.connect() })
+ }, 60_000)
+
+ afterAll(async () => {
+ await pool?.query('DELETE FROM rate_limit_buckets WHERE bucket_key = $1', [bucketKey])
+ await pool?.end()
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ it('returns distinct post-increment counts under concurrent requests', async () => {
+ const results = await Promise.all(
+ Array.from({ length: 32 }, () =>
+ checkAndIncrementWithQuery(
+ (text, values) => pool.query(text, values),
+ bucketKey,
+ 32,
+ windowStartMs,
+ 1
+ )
+ )
+ )
+ const counts = results.map(result => result.count).sort((a, b) => a - b)
+ expect(counts).toEqual(Array.from({ length: 32 }, (_, index) => index + 1))
+ const persisted = await pool.query<{ count: string }>(
+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1 AND window_start_ms = $2',
+ [bucketKey, windowStartMs]
+ )
+ expect(Number(persisted.rows[0]?.count)).toBe(32)
+ })
+
+ it('serializes a bounded advisory slot across independent PostgreSQL sessions', async () => {
+ const key = `gfs-upload-slot:${randomBytes(8).toString('hex')}`
+ const first = await pool.connect()
+ const second = await pool.connect()
+ try {
+ const acquiredFirst = await first.query<{ acquired: boolean }>(
+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
+ [key]
+ )
+ const acquiredSecond = await second.query<{ acquired: boolean }>(
+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
+ [key]
+ )
+ expect(acquiredFirst.rows[0]?.acquired).toBe(true)
+ expect(acquiredSecond.rows[0]?.acquired).toBe(false)
+
+ await first.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])
+ const acquiredAfterRelease = await second.query<{ acquired: boolean }>(
+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
+ [key]
+ )
+ expect(acquiredAfterRelease.rows[0]?.acquired).toBe(true)
+ await second.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])
+ } finally {
+ first.release()
+ second.release()
+ }
+ })
+})
diff --git a/control-ui/components/GfsBrowser.tsx b/control-ui/components/GfsBrowser.tsx
index 29210735b..86df75d0d 100644
--- a/control-ui/components/GfsBrowser.tsx
+++ b/control-ui/components/GfsBrowser.tsx
@@ -173,6 +173,44 @@ export function GfsBrowser(): React.JSX.Element {
// Resource IDs currently streaming a download (disables that row's button).
const [downloadingIds, setDownloadingIds] = useState<ReadonlySet<string>>(() => new Set())
+ /**
+ * The uploader already emits the exact aggregate of committed bytes plus
+ * each part's maximum in-flight observation. Keep a small monotonic guard
+ * only for two stale `uploading` snapshots arriving out of order; terminal
+ * or paused snapshots, and direct progress observations, must be accepted
+ * exactly so a failed retry cannot leave the bar overstated.
+ */
+ const mergeUploadSnapshot = useCallback((next: GfsUploadJobSnapshot): void => {
+ setUploadSnapshot(previous => {
+ if (!previous) return next
+ const previousUploadId = previous.session?.uploadId
+ const nextUploadId = next.session?.uploadId
+ if (
+ (previousUploadId && nextUploadId && previousUploadId !== nextUploadId) ||
+ previous.totalBytes !== next.totalBytes
+ )
+ return next
+ const uploadedBytes =
+ previous.state === 'uploading' && next.state === 'uploading'
+ ? Math.max(previous.uploadedBytes, next.uploadedBytes)
+ : next.uploadedBytes
+ return {
+ ...next,
+ uploadedBytes: Math.min(Math.max(0, uploadedBytes), next.totalBytes),
+ }
+ })
+ }, [])
+
+ const mergeUploadProgress = useCallback((uploadedBytes: number): void => {
+ setUploadSnapshot(previous => {
+ if (!previous) return previous
+ return {
+ ...previous,
+ uploadedBytes: Math.min(Math.max(0, uploadedBytes), previous.totalBytes),
+ }
+ })
+ }, [])
+
const current = crumbs[crumbs.length - 1]
const currentLabel = current?.name === '/' ? DRIVE : current?.name || DRIVE
@@ -284,13 +322,11 @@ export function GfsBrowser(): React.JSX.Element {
const job = createGfsUploadJob({
...input,
onProgress: progress => {
- setUploadSnapshot(previous =>
- previous ? { ...previous, uploadedBytes: progress.uploadedBytes } : previous
- )
+ mergeUploadProgress(progress.uploadedBytes)
input.onProgress?.(progress)
},
onState: snapshot => {
- setUploadSnapshot(snapshot)
+ mergeUploadSnapshot(snapshot)
input.onState?.(snapshot)
},
onPersistPending: record => {
@@ -412,11 +448,9 @@ export function GfsBrowser(): React.JSX.Element {
})
},
onProgress: progress => {
- setUploadSnapshot(previous =>
- previous ? { ...previous, uploadedBytes: progress.uploadedBytes } : previous
- )
+ mergeUploadProgress(progress.uploadedBytes)
},
- onState: snapshot => setUploadSnapshot(snapshot),
+ onState: snapshot => mergeUploadSnapshot(snapshot),
onPersist: record => {
persistPendingGfsUpload({
uploadId: record.uploadId,
diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsx
index dabf15c80..57d3f5e14 100644
--- a/control-ui/components/__tests__/GfsBrowser.test.tsx
+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx
@@ -367,9 +367,23 @@ describe('GfsBrowser', () => {
}
const paused = { ...completed, state: 'paused', committedBytes: 2 }
let resolveStart: ((value: typeof completed) => void) | undefined
- let uploadInput: { onState?: (snapshot: unknown) => void } | undefined
+ let uploadInput:
+ | {
+ onState?: (snapshot: unknown) => void
+ onProgress?: (progress: { uploadedBytes: number }) => void
+ }
+ | undefined
const job = {
start: vi.fn(() => {
+ uploadInput?.onState?.({
+ state: 'uploading',
+ session: { ...completed, state: 'uploading' },
+ uploadedBytes: 2,
+ totalBytes: 4,
+ })
+ // Simulate an out-of-order in-flight part reporting a later absolute
+ // offset before the accumulator commits the lower-numbered part.
+ uploadInput?.onProgress?.({ uploadedBytes: 3 })
uploadInput?.onState?.({
state: 'uploading',
session: { ...completed, state: 'uploading' },
@@ -408,7 +422,10 @@ describe('GfsBrowser', () => {
cancel: vi.fn(async () => undefined),
}
mockCreateGfsUploadJob.mockImplementationOnce(
- (input: { onState?: (snapshot: unknown) => void }) => {
+ (input: {
+ onState?: (snapshot: unknown) => void
+ onProgress?: (progress: { uploadedBytes: number }) => void
+ }) => {
uploadInput = input
return job
}
@@ -424,10 +441,10 @@ describe('GfsBrowser', () => {
const progress = await within(uploadDialog).findByRole('progressbar', {
name: /Upload progress/,
})
- expect(progress).toHaveAttribute('value', '2')
+ expect(progress).toHaveAttribute('value', '3')
expect(progress).toHaveAttribute('aria-valuemin', '0')
expect(progress).toHaveAttribute('aria-valuemax', '4')
- expect(progress).toHaveAttribute('aria-valuenow', '2')
+ expect(progress).toHaveAttribute('aria-valuenow', '3')
fireEvent.click(within(uploadDialog).getByRole('button', { name: 'Pause' }))
await waitFor(() => expect(job.pause).toHaveBeenCalledTimes(1))
expect(await within(uploadDialog).findByRole('button', { name: 'Resume' })).toBeTruthy()
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index d21787b80..eac70e826 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -1,6 +1,11 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { GFS_FILE_UPLOAD_MAX_BYTES, GFS_FILE_UPLOAD_MAX_MEGABYTES } from '@constants/gfsFileUpload'
-import { GfsUploadJob, type GfsUploadStatus, assertGfsFileUploadSize } from '@lib/gfsFileUpload'
+import {
+ GfsUploadJob,
+ type GfsUploadStatus,
+ assertGfsFileUploadSize,
+ parseRetryAfter,
+} from '@lib/gfsFileUpload'
describe('assertGfsFileUploadSize', () => {
it('accepts files at the upload limit', () => {
@@ -14,6 +19,14 @@ describe('assertGfsFileUploadSize', () => {
})
})
+describe('parseRetryAfter', () => {
+ it('accepts delta-seconds and caps untrusted server delays', () => {
+ expect(parseRetryAfter('2')).toBe(2_000)
+ expect(parseRetryAfter('60')).toBe(5_000)
+ expect(parseRetryAfter('not-a-date')).toBeUndefined()
+ })
+})
+
class FakeXhr {
static requests: FakeXhr[] = []
static statuses: number[] = []
@@ -32,9 +45,13 @@ class FakeXhr {
setRequestHeader(name: string, value: string): void {
this.headers.set(name, value)
}
+ getResponseHeader(name: string): string | null {
+ return this.headers.get(name) ?? null
+ }
send(body: Blob): void {
this.body = body
this.status = FakeXhr.statuses.shift() ?? 204
+ if (this.status === 429) this.headers.set('retry-after', '0')
FakeXhr.requests.push(this)
void body.arrayBuffer().then(() => {
this.upload.onprogress?.({
@@ -68,6 +85,78 @@ describe('GfsUploadJob', () => {
vi.restoreAllMocks()
})
+ it('honors the writer-advertised maxFileBytes before creating a session', async () => {
+ const file = new File([new Uint8Array([1, 2])], 'ceiling.bin')
+ let createCalls = 0
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ if (method === 'GET' && String(_input).endsWith('/capabilities'))
+ return new Response(
+ JSON.stringify({ upload: { resumableV2: { enabled: true, maxFileBytes: 1 } } }),
+ { status: 200 }
+ )
+ if (method === 'POST' && String(_input).endsWith('/uploads')) createCalls += 1
+ return new Response(null, { status: 204 })
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-ceiling' },
+ }).start()
+ ).rejects.toThrow('writer limit is 1 bytes')
+ expect(createCalls).toBe(0)
+ })
+
+ it('rejects a create receipt that changes the canonical control-ui drive', async () => {
+ const file = new File([new Uint8Array([3])], 'drive-drift.bin')
+ let partCalls = 0
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(
+ JSON.stringify({
+ ok: true,
+ data: {
+ uploadId: '18181818-1818-4181-8181-181818181818',
+ drive: 'archive',
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ },
+ }),
+ { status: 201 }
+ )
+ if (method === 'PUT') partCalls += 1
+ return new Response(null, { status: 204 })
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-drive-drift' },
+ }).start()
+ ).rejects.toThrow('upload_drive_mismatch')
+ expect(partCalls).toBe(0)
+ })
+
it('resumes only the missing part and emits visible progress before completion', async () => {
const file = new File([new Uint8Array([1, 2, 3, 4])], 'payload.bin', { lastModified: 12 })
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', new Uint8Array([1, 2])))
@@ -139,6 +228,106 @@ describe('GfsUploadJob', () => {
expect(responses).toHaveLength(0)
})
+ it('retries a lifecycle 429 using the server Retry-After header', async () => {
+ const file = new File([new Uint8Array([21])], 'retry-after.bin')
+ const session = {
+ uploadId: '12121212-1212-4121-8121-121212121212',
+ drive: 'main',
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ let createCalls = 0
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads')) {
+ createCalls += 1
+ if (createCalls === 1)
+ return new Response('{"error":"quota_exceeded"}', {
+ status: 429,
+ headers: { 'retry-after': '0' },
+ })
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ }
+ if (method === 'POST' && url.endsWith('/complete'))
+ return new Response(
+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),
+ {
+ status: 200,
+ }
+ )
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-retry-after' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(createCalls).toBe(2)
+ })
+
+ it('retries an XHR part 429 using the server Retry-After header', async () => {
+ const file = new File([new Uint8Array([22])], 'part-retry-after.bin')
+ const session = {
+ uploadId: '13131313-1313-4131-8131-131313131314',
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ FakeXhr.statuses = [429, 204]
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'POST' && url.endsWith('/complete'))
+ return new Response(
+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),
+ {
+ status: 200,
+ }
+ )
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-part-retry-after' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(FakeXhr.requests).toHaveLength(2)
+ })
+
it('adopts a status-confirmed committed part after its XHR response is lost', async () => {
const file = new File([new Uint8Array([5, 6, 7, 8])], 'response-loss.bin', {
lastModified: 13,
@@ -416,8 +605,8 @@ describe('GfsUploadJob', () => {
upload: {
resumableV2: {
enabled: true,
- preferredPartBytes: 2,
- maxPartBytes: 2,
+ preferredChunkBytes: 2,
+ maxChunkBytes: 2,
maxConcurrentPartsPerSession: 4,
fallbackConcurrency: 2,
},
@@ -508,7 +697,7 @@ describe('GfsUploadJob', () => {
if (init?.method === 'GET' && String(_input).endsWith('/capabilities'))
return new Response(
JSON.stringify({
- upload: { resumableV2: { enabled: true, preferredPartBytes: 2, maxPartBytes: 2 } },
+ upload: { resumableV2: { enabled: true, preferredChunkBytes: 2, maxChunkBytes: 2 } },
}),
{ status: 200 }
)
diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.ts
index dec63d25f..7734ec689 100644
--- a/control-ui/lib/gfsFileUpload.ts
+++ b/control-ui/lib/gfsFileUpload.ts
@@ -8,7 +8,11 @@ import {
import { apiSend } from './api'
const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'
+const GFS_UPLOAD_DRIVE = 'main'
const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
+export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
+export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
+const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000
const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60_000
const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
@@ -108,8 +112,7 @@ interface UploadCapabilities {
upload?: {
resumableV2?: {
enabled?: boolean
- preferredPartBytes?: number
- maxPartBytes?: number
+ maxFileBytes?: number
preferredChunkBytes?: number
maxChunkBytes?: number
maxConcurrentPartsPerSession?: number
@@ -193,7 +196,7 @@ async function requestJson<T>(
async function uploadError(
response: Response
-): Promise<Error & { status?: number; code?: string }> {
+): Promise<Error & { status?: number; code?: string; retryAfterMs?: number }> {
const text = await response.text().catch(() => '')
let message = text || response.statusText || 'GFS upload failed'
let code: string | undefined
@@ -218,19 +221,44 @@ async function uploadError(
const error = new Error(`${response.status} ${message}`) as Error & {
status?: number
code?: string
+ retryAfterMs?: number
}
error.status = response.status
error.code = code
+ error.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'))
return error
}
+/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */
+export function parseRetryAfter(
+ value: string | null | undefined,
+ nowMs = Date.now(),
+ capMs = GFS_UPLOAD_RETRY_AFTER_CAP_MS
+): number | undefined {
+ if (value === null || value === undefined) return undefined
+ const trimmed = value.trim()
+ if (!trimmed) return undefined
+ const seconds = Number(trimmed)
+ let delayMs: number
+ if (Number.isFinite(seconds) && seconds >= 0) {
+ delayMs = seconds * 1_000
+ } else {
+ const dateMs = Date.parse(trimmed)
+ if (!Number.isFinite(dateMs)) return undefined
+ delayMs = Math.max(0, dateMs - nowMs)
+ }
+ if (!Number.isFinite(delayMs) || delayMs < 0) return undefined
+ return Math.min(capMs, delayMs)
+}
+
function assertRetryable(error: unknown): boolean {
if (
error instanceof Error &&
(error.name === 'AbortError' ||
error.message.startsWith('source_changed:') ||
error.message.startsWith('upload_part_mismatch:') ||
- error.message.startsWith('upload_part_outcome_unknown:'))
+ error.message.startsWith('upload_part_outcome_unknown:') ||
+ error.message.startsWith('upload_completion_outcome_unknown:'))
)
return false
const status =
@@ -256,6 +284,92 @@ function delay(ms: number, signal?: AbortSignal): Promise<void> {
})
}
+function retryAfterOf(error: unknown): number | undefined {
+ if (!error || typeof error !== 'object') return undefined
+ const value = error as {
+ retryAfterMs?: unknown
+ retryAfter?: unknown
+ headers?: { get?: (name: string) => string | null }
+ response?: { headers?: { get?: (name: string) => string | null } }
+ }
+ if (typeof value.retryAfterMs === 'number' && Number.isFinite(value.retryAfterMs))
+ return Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, Math.max(0, value.retryAfterMs))
+ const header =
+ typeof value.retryAfter === 'string'
+ ? value.retryAfter
+ : (value.headers?.get?.('retry-after') ?? value.response?.headers?.get?.('retry-after'))
+ return parseRetryAfter(typeof header === 'string' ? header : undefined)
+}
+
+function ambiguousLifecycleError(error: unknown): boolean {
+ const status = statusOf(error)
+ return status === undefined || status === 408 || (status !== undefined && status >= 500)
+}
+
+function terminalReconciliationError(error: unknown): boolean {
+ if (!(error instanceof Error)) return false
+ return [
+ 'upload status changed during reconciliation',
+ 'upload_part_mismatch:',
+ 'source_changed:',
+ 'upload_completion_outcome_unknown:',
+ 'cannot reconcile ',
+ ].some(prefix => error.message.startsWith(prefix))
+}
+
+interface LifecycleRecovery<T> {
+ value: T
+}
+
+interface LifecycleRetryOptions<T> {
+ signal?: AbortSignal
+ attempts?: number
+ reconcileOnError?: (error: unknown) => boolean
+ reconcile?: (error: unknown) => Promise<LifecycleRecovery<T> | undefined>
+}
+
+/**
+ * Bounded lifecycle retry primitive. It centralizes status classification,
+ * capped Retry-After handling, abort propagation, and optional ambiguity
+ * reconciliation for state-changing requests.
+ */
+async function requestWithLifecycleRetries<T>(
+ operation: () => Promise<T>,
+ options: LifecycleRetryOptions<T> = {}
+): Promise<T> {
+ const attempts = Math.max(1, options.attempts ?? GFS_UPLOAD_RETRY_MAX_ATTEMPTS)
+ let lastError: unknown
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
+ try {
+ return await operation()
+ } catch (error) {
+ lastError = error
+ if (options.signal?.aborted) throw options.signal.reason ?? error
+ const shouldReconcile = options.reconcile
+ ? (options.reconcileOnError?.(error) ?? ambiguousLifecycleError(error))
+ : false
+ if (shouldReconcile) {
+ try {
+ const recovered = await options.reconcile!(error)
+ if (recovered) return recovered.value
+ } catch (reconcileError) {
+ // A transient status/HEAD failure must not suppress the bounded
+ // retry of the original request. Definitive geometry, auth, or
+ // outcome-unknown errors remain terminal and are surfaced.
+ if (terminalReconciliationError(reconcileError)) throw reconcileError
+ }
+ }
+ if (!assertRetryable(error) || attempt + 1 >= attempts) throw error
+ await delay(
+ retryAfterOf(error) ??
+ Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ options.signal
+ )
+ }
+ }
+ throw lastError
+}
+
function digestBase64(bytes: ArrayBuffer): string {
let binary = ''
const view = new Uint8Array(bytes)
@@ -309,16 +423,21 @@ function indexCommittedParts(
async function loadUploadStatus(
uploadId: string,
- expected: { expectedBytes: number; partBytes?: number; partCount?: number },
+ expected: { expectedBytes: number; drive?: string; partBytes?: number; partCount?: number },
signal?: AbortSignal,
- timeoutMs = 600_000
+ timeoutMs = 600_000,
+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS
): Promise<{ status: GfsUploadStatus; head: Response }> {
- const head = await requestJson<undefined>(
- 'HEAD',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}`,
- undefined,
- signal,
- timeoutMs
+ const head = await requestWithLifecycleRetries(
+ () =>
+ requestJson<undefined>(
+ 'HEAD',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}`,
+ undefined,
+ signal,
+ timeoutMs
+ ),
+ { signal, attempts: retryAttempts }
)
const declaredLengthHeader = head.response.headers.get('upload-length')
if (declaredLengthHeader !== null) {
@@ -332,12 +451,16 @@ async function loadUploadStatus(
const parts: GfsUploadPartReceipt[] = []
for (let page = 0; page < 8; page += 1) {
const query = cursor ? `?limit=256&cursor=${encodeURIComponent(cursor)}` : '?limit=256'
- const response = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
- 'GET',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}/status${query}`,
- undefined,
- signal,
- timeoutMs
+ const response = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: GfsUploadStatus }>(
+ 'GET',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}/status${query}`,
+ undefined,
+ signal,
+ timeoutMs
+ ),
+ { signal, attempts: retryAttempts }
)
const current = receiptFromResponse(response)
session ??= current.session
@@ -346,6 +469,9 @@ async function loadUploadStatus(
session.uploadId !== uploadId ||
session.expectedBytes !== current.session.expectedBytes ||
session.expectedBytes !== expected.expectedBytes ||
+ (expected.drive !== undefined &&
+ current.session.drive !== undefined &&
+ current.session.drive !== expected.drive) ||
session.partBytes !== current.session.partBytes ||
session.partCount !== current.session.partCount ||
(expected.partBytes !== undefined && current.session.partBytes !== expected.partBytes) ||
@@ -388,11 +514,13 @@ async function reconcileAmbiguousPart(
session.uploadId,
{
expectedBytes: file.size,
+ drive: GFS_UPLOAD_DRIVE,
partBytes: session.partBytes,
partCount: session.partCount,
},
signal,
- GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,
+ 1
)
).status
} catch (error) {
@@ -475,8 +603,9 @@ async function putPart(
}
const error = new Error(
`${xhr.status} ${xhr.responseText || 'GFS upload failed'}`
- ) as Error & { status?: number }
+ ) as Error & { status?: number; retryAfterMs?: number }
error.status = xhr.status
+ error.retryAfterMs = parseRetryAfter(xhr.getResponseHeader('retry-after'))
reject(error)
}
xhr.onerror = () => reject(new Error('GFS upload network error'))
@@ -534,7 +663,10 @@ async function uploadPartWithRetries(
}
}
if (attempt === 2) throw error
- await delay(Math.min(1000, 150 * (attempt + 1)), signal)
+ await delay(
+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ signal
+ )
}
}
throw lastError
@@ -549,6 +681,7 @@ async function runParts(
shouldPause: () => boolean,
onProgress?: (progress: GfsUploadProgress) => void,
onByteProgress?: (progress: GfsUploadProgress) => void,
+ onPartFailure?: (partNumber: number) => void,
fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY
): Promise<{ failed: typeof parts; retryableFailures: number; paused: boolean }> {
const initialConcurrency = Math.max(1, Math.min(concurrency, parts.length || 1))
@@ -607,6 +740,7 @@ async function runParts(
failed.delete(part.partNumber)
retryableFailed.delete(part.partNumber)
} catch (error) {
+ onPartFailure?.(part.partNumber)
if (shouldPause()) {
return
}
@@ -650,6 +784,11 @@ function statusOf(error: unknown): number | undefined {
return value && typeof value.status === 'number' ? value.status : undefined
}
+function errorCodeOf(error: unknown): string | undefined {
+ const value = error as { code?: unknown } | null
+ return value && typeof value.code === 'string' ? value.code : undefined
+}
+
export class GfsUploadJob {
private readonly abortController = new AbortController()
private readonly idempotencyKey = uuid()
@@ -700,13 +839,19 @@ export class GfsUploadJob {
throw new Error(`cannot pause upload in ${session.state}`)
}
this.pauseRequested = true
- const response = await requestJson<{ ok: boolean; data: UploadReceipt }>(
- 'POST',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/pause`,
- {},
- undefined
+ const paused = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: UploadReceipt }>(
+ 'POST',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/pause`,
+ {},
+ this.abortController.signal
+ ).then(receiptFromResponse),
+ {
+ signal: this.abortController.signal,
+ reconcile: () => this.reconcileMutation(session, 'pause'),
+ }
)
- const paused = receiptFromResponse(response)
this.session = paused
this.state = 'paused'
this.emit()
@@ -721,13 +866,20 @@ export class GfsUploadJob {
throw new Error(`cannot resume upload in ${session.state}`)
}
this.pauseRequested = false
- const response = await requestJson<{ ok: boolean; data: UploadReceipt }>(
- 'POST',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/resume`,
- {},
- this.abortController.signal
+ const resumed = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: UploadReceipt }>(
+ 'POST',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/resume`,
+ {},
+ this.abortController.signal
+ ).then(receiptFromResponse),
+ {
+ signal: this.abortController.signal,
+ reconcile: () => this.reconcileMutation(session, 'resume'),
+ }
)
- this.session = receiptFromResponse(response)
+ this.session = resumed
this.state = 'uploading'
this.emit()
return this.start()
@@ -741,40 +893,52 @@ export class GfsUploadJob {
this.state = 'canceling'
this.emit()
if (session && !['completed', 'aborted'].includes(session.state)) {
+ let reconciledCompleted: UploadReceipt | null = null
try {
- await requestJson(
- 'DELETE',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}`,
- undefined,
- undefined
+ await requestWithLifecycleRetries(
+ () =>
+ requestJson(
+ 'DELETE',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}`,
+ undefined,
+ undefined
+ ).then(() => undefined),
+ {
+ reconcileOnError: error =>
+ ambiguousLifecycleError(error) ||
+ statusOf(error) === 409 ||
+ errorCodeOf(error) === 'upload_finalizing',
+ reconcile: async error => {
+ const observed = await this.observeSession(
+ session,
+ GFS_UPLOAD_RETRY_MAX_ATTEMPTS,
+ undefined
+ )
+ this.session = observed
+ if (observed.state === 'completed') {
+ reconciledCompleted = observed
+ return { value: undefined }
+ }
+ if (observed.state === 'aborted') return { value: undefined }
+ if (observed.state === 'finalizing') throw error
+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined
+ throw new Error(`cannot reconcile cancel upload in ${observed.state}`)
+ },
+ }
)
} catch (error) {
- if (
- statusOf(error) === 409 ||
- (typeof error === 'object' &&
- error !== null &&
- (error as { code?: unknown }).code === 'upload_finalizing')
- ) {
- const reconciled = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
- 'GET',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/status`,
- undefined,
- undefined
- )
- this.session = receiptFromResponse(reconciled).session
- if (this.session.state === 'completed') {
- this.state = 'completed'
- this.canceled = false
- this.emit()
- await this.input.onClearPersisted?.(session.uploadId)
- return
- }
- }
this.canceled = false
this.state = 'failed'
this.emit()
throw error
}
+ if (reconciledCompleted) {
+ this.state = 'completed'
+ this.canceled = false
+ this.emit()
+ await this.input.onClearPersisted?.(session.uploadId)
+ return
+ }
}
await this.runPromise?.catch(() => undefined)
this.state = 'aborted'
@@ -800,14 +964,76 @@ export class GfsUploadJob {
}
}
+ private async observeSession(
+ session: UploadReceipt,
+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS,
+ signal: AbortSignal | undefined = this.abortController.signal
+ ): Promise<UploadReceipt> {
+ return (
+ await loadUploadStatus(
+ session.uploadId,
+ {
+ expectedBytes: this.input.file.size,
+ drive: GFS_UPLOAD_DRIVE,
+ partBytes: session.partBytes,
+ partCount: session.partCount,
+ },
+ signal,
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,
+ retryAttempts
+ )
+ ).status.session
+ }
+
+ private async reconcileMutation(
+ session: UploadReceipt,
+ action: 'pause' | 'resume'
+ ): Promise<LifecycleRecovery<UploadReceipt> | undefined> {
+ const observed = await this.observeSession(session)
+ if (
+ (action === 'pause' && observed.state === 'paused') ||
+ (action === 'resume' && ['initiated', 'uploading', 'completed'].includes(observed.state))
+ ) {
+ return { value: observed }
+ }
+ if (
+ (action === 'pause' && ['initiated', 'uploading'].includes(observed.state)) ||
+ (action === 'resume' && observed.state === 'paused')
+ ) {
+ return undefined
+ }
+ throw new Error(`cannot reconcile ${action} upload in ${observed.state}`)
+ }
+
+ private async reconcileCompletion(
+ session: UploadReceipt
+ ): Promise<LifecycleRecovery<UploadReceipt> | undefined> {
+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {
+ const observed = await this.observeSession(session, 1)
+ if (observed.state === 'completed') return { value: observed }
+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined
+ if (observed.state !== 'finalizing')
+ throw new Error(`cannot reconcile upload completion in ${observed.state}`)
+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)
+ await delay(GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt, this.abortController.signal)
+ }
+ throw new Error(
+ 'upload_completion_outcome_unknown: finalization is still in progress; resume the persisted session'
+ )
+ }
+
private async loadCapabilities(): Promise<UploadCapabilities['upload']> {
let capabilities: { data: UploadCapabilities }
try {
- capabilities = await requestJson<UploadCapabilities>(
- 'GET',
- '/api/v1/gfs/proxy/v1/capabilities',
- undefined,
- this.abortController.signal
+ capabilities = await requestWithLifecycleRetries(
+ () =>
+ requestJson<UploadCapabilities>(
+ 'GET',
+ '/api/v1/gfs/proxy/v1/capabilities',
+ undefined,
+ this.abortController.signal
+ ),
+ { signal: this.abortController.signal }
)
} catch (error) {
throw new GfsUploadCapabilityError('Resumable GFS upload capabilities are unavailable.', {
@@ -827,12 +1053,20 @@ export class GfsUploadJob {
}> {
const upload = await this.loadCapabilities()
const resumable = upload.resumableV2!
+ if (resumable.maxFileBytes !== undefined) {
+ if (!Number.isSafeInteger(resumable.maxFileBytes) || resumable.maxFileBytes < 0)
+ throw new GfsUploadCapabilityError(
+ 'Resumable GFS upload capabilities advertised an invalid file ceiling.'
+ )
+ if (this.input.file.size > resumable.maxFileBytes)
+ throw new Error(`GFS writer limit is ${resumable.maxFileBytes} bytes for this upload.`)
+ }
const resumeId = this.session?.uploadId ?? this.input.resumeUploadId
if (resumeId) {
const { session, parts } = (
await loadUploadStatus(
resumeId,
- { expectedBytes: this.input.file.size },
+ { expectedBytes: this.input.file.size, drive: GFS_UPLOAD_DRIVE },
this.abortController.signal
)
).status
@@ -858,27 +1092,21 @@ export class GfsUploadJob {
target: this.input.target,
name: this.input.name,
})
- let create: { data: { ok: boolean; data: UploadReceipt }; response: Response }
- try {
- create = await requestJson<{ ok: boolean; data: UploadReceipt }>(
- 'POST',
- '/api/v1/gfs/proxy/v1/uploads',
- createBody,
- this.abortController.signal
- )
- } catch (error) {
- // A response lost after POST is safe to replay because the key is fixed.
- if (statusOf(error) !== undefined) throw error
- create = await requestJson<{ ok: boolean; data: UploadReceipt }>(
- 'POST',
- '/api/v1/gfs/proxy/v1/uploads',
- createBody,
- this.abortController.signal
- )
- }
+ const create = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: UploadReceipt }>(
+ 'POST',
+ '/api/v1/gfs/proxy/v1/uploads',
+ createBody,
+ this.abortController.signal
+ ),
+ { signal: this.abortController.signal }
+ )
const session = receiptFromResponse(create)
if (session.expectedBytes !== this.input.file.size)
throw new Error('GFS upload session size changed before transfer')
+ if (session.drive !== undefined && session.drive !== GFS_UPLOAD_DRIVE)
+ throw new Error('upload_drive_mismatch: writer changed drive')
return { session, committed: new Set<number>(), resumable }
}
@@ -938,12 +1166,49 @@ export class GfsUploadJob {
}))
const allParts = parts
const completedParts = new Set(prepared.committed)
- const progress = (value: GfsUploadProgress) => {
- completedParts.add(value.partNumber)
- this.uploadedBytes = allParts.reduce(
+ const inFlightLoaded = new Map<number, number>()
+ const calculateUploadedBytes = (): number => {
+ const committedBytes = allParts.reduce(
(total, part) => total + (completedParts.has(part.partNumber) ? part.lengthBytes : 0),
0
)
+ const inFlightBytes = [...inFlightLoaded.entries()].reduce(
+ (total, [partNumber, loaded]) => total + (completedParts.has(partNumber) ? 0 : loaded),
+ 0
+ )
+ const observed = Math.min(this.input.file.size, committedBytes + inFlightBytes)
+ // Part bytes are committed before the final resource publication. Do
+ // not expose a visually complete bar until the complete receipt is
+ // durable; the caller sets the exact total only after that response.
+ return Math.min(Math.max(0, this.input.file.size - 1), observed)
+ }
+ const progress = (value: GfsUploadProgress) => {
+ completedParts.add(value.partNumber)
+ inFlightLoaded.delete(value.partNumber)
+ this.uploadedBytes = calculateUploadedBytes()
+ this.input.onProgress?.({
+ ...value,
+ uploadedBytes: this.uploadedBytes,
+ partCount: allParts.length,
+ })
+ this.emit()
+ }
+ const byteProgress = (value: GfsUploadProgress) => {
+ const part = allParts[value.partNumber]
+ if (!part) return
+ const loaded = Math.min(
+ part.lengthBytes,
+ Math.max(0, value.uploadedBytes - part.offsetBytes)
+ )
+ inFlightLoaded.set(
+ value.partNumber,
+ Math.max(inFlightLoaded.get(value.partNumber) ?? 0, loaded)
+ )
+ // A progress event is an observation of bytes sent, not a commit
+ // receipt. Deduplicate each part's in-flight contribution, and remove
+ // it when the worker reports a failed attempt so a retry cannot leave
+ // the aggregate permanently overstated.
+ this.uploadedBytes = calculateUploadedBytes()
this.input.onProgress?.({
...value,
uploadedBytes: this.uploadedBytes,
@@ -967,30 +1232,26 @@ export class GfsUploadJob {
this.abortController.signal,
() => this.pauseRequested || this.canceled,
progress,
- value => this.input.onProgress?.(value),
+ byteProgress,
+ partNumber => {
+ inFlightLoaded.delete(partNumber)
+ this.uploadedBytes = calculateUploadedBytes()
+ },
fallbackConcurrency
)
if (result.paused || this.pauseRequested) {
this.state = 'paused'
- const paused = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
- 'GET',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,
- undefined,
- undefined
- )
- this.session = receiptFromResponse(paused).session
- this.emit()
- return this.session
- }
- if (result.paused || this.pauseRequested) {
- this.state = 'paused'
- const paused = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(
- 'GET',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,
- undefined,
- undefined
+ const paused = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: GfsUploadStatus }>(
+ 'GET',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,
+ undefined,
+ this.abortController.signal
+ ).then(receiptFromResponse),
+ { signal: this.abortController.signal }
)
- this.session = receiptFromResponse(paused).session
+ this.session = paused.session
this.emit()
return this.session
}
@@ -998,13 +1259,22 @@ export class GfsUploadJob {
throw new Error('One or more upload parts could not be committed.')
this.state = 'finalizing'
this.emit()
- const completedResponse = await requestJson<{ ok: boolean; data: UploadReceipt }>(
- 'POST',
- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`,
- {},
- this.abortController.signal
+ const completed = await requestWithLifecycleRetries(
+ () =>
+ requestJson<{ ok: boolean; data: UploadReceipt }>(
+ 'POST',
+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`,
+ {},
+ this.abortController.signal
+ ).then(receiptFromResponse),
+ {
+ signal: this.abortController.signal,
+ reconcileOnError: error =>
+ ambiguousLifecycleError(error) || errorCodeOf(error) === 'upload_finalizing',
+ reconcile: () => this.reconcileCompletion(prepared.session),
+ }
)
- this.session = receiptFromResponse(completedResponse)
+ this.session = completed
this.state = 'completed'
this.uploadedBytes = this.input.file.size
this.input.onProgress?.({
diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.ts
index 2d6565e2c..fa12dbd6b 100644
--- a/desktop-app/src/__tests__/appService.gfs.test.ts
+++ b/desktop-app/src/__tests__/appService.gfs.test.ts
@@ -247,6 +247,12 @@ function authenticatedUploadService(statePath: string): UploadScopeTestService {
}
service.gfsAuthEpoch = 7
service.gfsDispatchBlocked = false
+ ;(service as any).gfsScopeIdentity = {
+ ownerId: service.me.id,
+ teamId: service.me.teamId,
+ environmentKey: getActiveEnvKey(),
+ baseUrl: canonicalExternalBaseUrl(),
+ }
service.desktopGfsUploadStatePath = async () => statePath
service.authClient = {
passwordLogin: vi.fn(),
@@ -430,6 +436,12 @@ describe('AppService GFS upload security scope', () => {
role: null,
}
service.gfsDispatchBlocked = false
+ ;(service as any).gfsScopeIdentity = {
+ ownerId: service.me.id,
+ teamId: service.me.teamId,
+ environmentKey: getActiveEnvKey(),
+ baseUrl: canonicalExternalBaseUrl(),
+ }
await expect(service.listGfsUploadSessions('main')).resolves.toEqual([])
await expect(service.resumeGfsUpload(exact.uploadId, 'main')).rejects.toThrow(
'not available in the active security scope'
@@ -447,6 +459,12 @@ describe('AppService GFS upload security scope', () => {
role: null,
}
service.gfsAuthEpoch += 1
+ ;(service as any).gfsScopeIdentity = {
+ ownerId: service.me.id,
+ teamId: service.me.teamId,
+ environmentKey: getActiveEnvKey(),
+ baseUrl: canonicalExternalBaseUrl(),
+ }
service.startDesktopGfsUpload.mockResolvedValue({ ...exact.session, state: 'uploading' })
await service.resumeGfsUpload(exact.uploadId, 'main')
expect(service.startDesktopGfsUpload).toHaveBeenCalledWith(
diff --git a/desktop-app/src/__tests__/appService.rpc.test.ts b/desktop-app/src/__tests__/appService.rpc.test.ts
index 83c6f9988..78272c6a6 100644
--- a/desktop-app/src/__tests__/appService.rpc.test.ts
+++ b/desktop-app/src/__tests__/appService.rpc.test.ts
@@ -147,6 +147,182 @@ describe('AppService.invokeHostMessage', () => {
)
})
+ it('does not fence GFS uploads during a transient cross-team operation hop', async () => {
+ const service = new AppService() as any
+ service.desktopGfsUploadStatePath = vi
+ .fn()
+ .mockResolvedValue(path.join(chatStoreBaseDir, 'gfs-upload-state.json'))
+ service.sessionToken = 'teamless-token'
+ service.me = {
+ id: 'user-1',
+ email: '[email protected]',
+ name: 'Test User',
+ picture: null,
+ teamId: 'team-a',
+ teamName: 'Team A',
+ role: 'member',
+ }
+ const uploadJob = { suspendForAuth: vi.fn() }
+ service.gfsUploadJobs = new Map([
+ [
+ 'upload-1',
+ {
+ job: uploadJob,
+ promise: Promise.resolve(undefined),
+ scope: {
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ drive: 'main',
+ authEpoch: 1,
+ },
+ },
+ ],
+ ])
+ service.gfsPendingUploadJobs = new Set()
+ service.gfsPendingLegacyUploads = new Set()
+ service.authClient = {
+ getMe: vi
+ .fn()
+ .mockResolvedValueOnce({ ...service.me, teamId: 'team-b', teamName: 'Team B' })
+ .mockResolvedValueOnce({ ...service.me }),
+ switchTeam: vi.fn(async (_token: string, teamId: string) => ({
+ token: `${teamId}-token`,
+ team: { id: teamId, name: teamId === 'team-b' ? 'Team B' : 'Team A', role: 'member' },
+ })),
+ }
+ service.tokenStore = { setSessionToken: vi.fn() }
+
+ await expect(
+ service.runWithTeamContext('team-b', async (token: string) => token)
+ ).resolves.toBe('team-b-token')
+
+ expect(uploadJob.suspendForAuth).not.toHaveBeenCalled()
+ expect(service.gfsUploadJobs).toHaveProperty('size', 1)
+ expect(service.me.teamId).toBe('team-a')
+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(1, 'teamless-token', 'team-b')
+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(2, 'team-b-token', 'team-a')
+ })
+
+ it('fences the old GFS scope if a replacement team token cannot be refreshed', async () => {
+ const service = new AppService() as any
+ service.sessionToken = 'team-a-token'
+ service.me = {
+ id: 'user-1',
+ email: '[email protected]',
+ name: 'Test User',
+ picture: null,
+ teamId: 'team-a',
+ teamName: 'Team A',
+ role: 'member',
+ }
+ service.gfsScopeIdentity = {
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ }
+ service.authClient = {
+ switchTeam: vi.fn().mockResolvedValue({
+ token: 'team-b-token',
+ team: { id: 'team-b', name: 'Team B', role: 'member' },
+ }),
+ getMe: vi.fn().mockRejectedValue(new Error('replacement session rejected')),
+ }
+ service.tokenStore = { setSessionToken: vi.fn(), clearSessionToken: vi.fn() }
+ service.rpcTokenManager = { clear: vi.fn() }
+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn().mockResolvedValue(undefined)
+ service.clearAuthenticatedSessionState = vi.fn()
+
+ await expect(service.switchSessionToTeam('team-b', 'team-a-token')).rejects.toThrow(
+ 'replacement session rejected'
+ )
+
+ expect(service.suspendDesktopGfsUploadsForAuthBoundary).toHaveBeenCalledWith({
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ })
+ expect(service.clearAuthenticatedSessionState).toHaveBeenCalledTimes(1)
+ })
+
+ it('fences GFS uploads only after an explicit switchTeam succeeds', async () => {
+ const service = new AppService() as any
+ service.sessionToken = 'team-a-token'
+ service.me = {
+ id: 'user-1',
+ email: '[email protected]',
+ name: 'Test User',
+ picture: null,
+ teamId: 'team-a',
+ teamName: 'Team A',
+ role: 'member',
+ }
+ service.gfsScopeIdentity = {
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ }
+ service.authClient = {
+ switchTeam: vi.fn().mockResolvedValue({
+ token: 'team-b-token',
+ team: { id: 'team-b', name: 'Team B', role: 'member' },
+ }),
+ getMe: vi.fn().mockResolvedValue({ ...service.me, teamId: 'team-b', teamName: 'Team B' }),
+ }
+ service.tokenStore = { setSessionToken: vi.fn() }
+ service.rpcTokenManager = { clear: vi.fn() }
+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn().mockResolvedValue(undefined)
+ service.activateGfsAuthScope = vi.fn()
+ service.stopAllStreams = vi.fn()
+ service.gfsUploadJobs = new Map([['upload-1', { job: { suspendForAuth: vi.fn() } }]])
+
+ await expect(service.switchTeam('team-b')).resolves.toMatchObject({
+ authenticated: true,
+ me: { teamId: 'team-b' },
+ })
+
+ expect(service.suspendDesktopGfsUploadsForAuthBoundary).toHaveBeenCalledWith({
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ })
+ expect(
+ service.suspendDesktopGfsUploadsForAuthBoundary.mock.invocationCallOrder[0]
+ ).toBeGreaterThan(service.authClient.switchTeam.mock.invocationCallOrder[0])
+ })
+
+ it('does not fence GFS uploads when an explicit switchTeam is rejected', async () => {
+ const service = new AppService() as any
+ service.sessionToken = 'team-a-token'
+ service.me = {
+ id: 'user-1',
+ email: '[email protected]',
+ name: 'Test User',
+ picture: null,
+ teamId: 'team-a',
+ teamName: 'Team A',
+ role: 'member',
+ }
+ service.gfsScopeIdentity = {
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ }
+ service.authClient = {
+ switchTeam: vi.fn().mockRejectedValue(new Error('team switch rejected')),
+ }
+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn()
+
+ await expect(service.switchTeam('team-b')).rejects.toThrow('team switch rejected')
+ expect(service.suspendDesktopGfsUploadsForAuthBoundary).not.toHaveBeenCalled()
+ })
+
it('uses the teamless session when a directly granted agent needs an RPC token', async () => {
const service = new AppService() as any
service.sessionToken = 'teamless-token'
diff --git a/desktop-app/src/appService.ts b/desktop-app/src/appService.ts
index c456986f7..29d57184d 100644
--- a/desktop-app/src/appService.ts
+++ b/desktop-app/src/appService.ts
@@ -298,7 +298,11 @@ const gfsUploadTransport: DesktopUploadTransport = {
signal: withTimeout(signal, timeoutMs),
} as RequestInit & { duplex: 'half' })
const text = await response.text()
- return { status: response.status, text }
+ return {
+ status: response.status,
+ text,
+ retryAfter: response.headers.get('retry-after') ?? undefined,
+ }
},
}
@@ -411,6 +415,8 @@ interface DesktopGfsUploadScope {
authEpoch: number
}
+type DesktopGfsUploadIdentity = Omit<DesktopGfsUploadScope, 'drive' | 'authEpoch'>
+
interface PersistedDesktopGfsUpload {
version: typeof DESKTOP_GFS_UPLOAD_STATE_VERSION
uploadId: string
@@ -473,6 +479,15 @@ function normalizeDesktopUploadBaseUrl(value: string): string {
return url.toString().replace(/\/+$/, '')
}
+function desktopGfsUploadIdentity(me: SessionMe): DesktopGfsUploadIdentity {
+ return {
+ ownerId: me.id,
+ teamId: me.teamId ? String(me.teamId).trim() : null,
+ environmentKey: getActiveEnvKey(),
+ baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),
+ }
+}
+
function normalizeDesktopUploadDrive(value: string | undefined): string {
const drive = String(value ?? 'main').trim()
if (!drive) throw new Error('drive_required: GFS uploads require a canonical drive')
@@ -803,6 +818,10 @@ export class AppService {
private logoutInProgress = false
private gfsAuthEpoch = 0
private gfsDispatchBlocked = true
+ // A finite RPC may temporarily borrow another team session. Keep the
+ // deliberate user-owned GFS scope stable across that hop so in-flight jobs
+ // are not invalidated by the transient `me` value.
+ private gfsScopeIdentity: DesktopGfsUploadIdentity | null = null
private gfsUploadStateQueue: Promise<void> = Promise.resolve()
private readonly gfsUploadJobs = new Map<
string,
@@ -975,11 +994,12 @@ export class AppService {
private async switchSessionToTeam(teamId: string, token = this.requireSessionToken()) {
const targetTeamId = String(teamId || '').trim()
if (!targetTeamId) throw new Error('teamId is required')
- await this.suspendDesktopGfsUploadsForAuthBoundary()
const switchGeneration = this.sessionGeneration
if (this.sessionToken !== token || !this.me) {
throw new Error('stale_auth_epoch: authenticated team scope changed before switch dispatch')
}
+ const previousIdentity =
+ this.gfsScopeIdentity ?? (this.me ? desktopGfsUploadIdentity(this.me) : undefined)
let switchedToken: string | null = null
try {
const switched = await this.authClient.switchTeam(token, targetTeamId)
@@ -989,12 +1009,17 @@ export class AppService {
}
await this.commitSessionToken(switched.token, { refreshMe: true })
if (!this.me) throw new Error('Team switch ended without an authenticated session')
- this.activateGfsAuthScope()
return switched.token
} catch (error) {
if (!switchedToken && this.sessionToken === token && this.me) {
- this.activateGfsAuthScope()
+ // A failed transient hop leaves the deliberate GFS scope untouched.
} else {
+ // Once the auth service has issued a replacement token, the previous
+ // authenticated scope is no longer safe to continue. Fence and
+ // persist its uploads before clearing the partially switched session;
+ // otherwise a failed refresh/restore could leave a job running with a
+ // token whose team ownership is unknown.
+ await this.suspendDesktopGfsUploadsForAuthBoundary(previousIdentity)
this.clearAuthenticatedSessionState()
try {
await this.tokenStore.clearSessionToken(getActiveEnvKey(), {
@@ -1132,6 +1157,7 @@ export class AppService {
this.sessionGeneration += 1
this.gfsAuthEpoch += 1
this.gfsDispatchBlocked = true
+ this.gfsScopeIdentity = null
this.stopAllStreams()
this.sessionToken = null
this.me = null
@@ -1144,20 +1170,22 @@ export class AppService {
unbindChatStore()
}
- private activateGfsAuthScope(): void {
+ private activateGfsAuthScope(identityOverride?: DesktopGfsUploadIdentity): void {
this.gfsAuthEpoch += 1
this.gfsDispatchBlocked = false
+ this.gfsScopeIdentity = identityOverride ?? (this.me ? desktopGfsUploadIdentity(this.me) : null)
}
private currentDesktopGfsUploadScope(driveValue: string | undefined): DesktopGfsUploadScope {
if (this.gfsDispatchBlocked || !this.sessionToken || !this.me) {
throw new Error('GFS upload dispatch is unavailable without an active authenticated scope')
}
+ const identity = this.gfsScopeIdentity
+ if (!identity) {
+ throw new Error('GFS upload dispatch is unavailable before scope activation')
+ }
return {
- ownerId: this.me.id,
- teamId: this.me.teamId ? String(this.me.teamId).trim() : null,
- environmentKey: getActiveEnvKey(),
- baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),
+ ...identity,
drive: normalizeDesktopUploadDrive(driveValue),
authEpoch: this.gfsAuthEpoch,
}
@@ -1170,15 +1198,22 @@ export class AppService {
}
}
- private async suspendDesktopGfsUploadsForAuthBoundary(): Promise<void> {
- const oldIdentity = this.me
- ? {
- ownerId: this.me.id,
- teamId: this.me.teamId ? String(this.me.teamId).trim() : null,
- environmentKey: getActiveEnvKey(),
- baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),
- }
- : null
+ private async suspendDesktopGfsUploadsForAuthBoundary(identityOverride?: {
+ ownerId: string
+ teamId: string | null
+ environmentKey: string
+ baseUrl: string
+ }): Promise<void> {
+ const oldIdentity =
+ identityOverride ??
+ (this.me
+ ? {
+ ownerId: this.me.id,
+ teamId: this.me.teamId ? String(this.me.teamId).trim() : null,
+ environmentKey: getActiveEnvKey(),
+ baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),
+ }
+ : null)
this.gfsDispatchBlocked = true
this.gfsAuthEpoch += 1
const jobs = new Set([
@@ -2739,6 +2774,8 @@ export class AppService {
async switchTeam(teamId: string): Promise<SessionState> {
const targetTeamId = String(teamId || '').trim()
if (!targetTeamId) throw new Error('teamId is required')
+ const previousIdentity =
+ this.gfsScopeIdentity ?? (this.me ? desktopGfsUploadIdentity(this.me) : undefined)
// §4.5-3 (GAP-N4 main half): a deliberate team switch tears down every live
// stream — the old team's progress/activity/status/notification sockets must
// not survive it. Done AFTER the switch SUCCEEDS (silently), mirroring the
@@ -2751,6 +2788,11 @@ export class AppService {
// minting a cross-team RPC token to decide an approval) must NOT.
await this.switchSessionToTeam(targetTeamId)
if (!this.me) throw new Error('Team switch ended without an authenticated session')
+ // A deliberate user switch is the only team-context boundary that fences
+ // in-flight uploads. Transient runWithTeamContext hops intentionally use
+ // switchSessionToTeam directly and must leave those jobs untouched.
+ await this.suspendDesktopGfsUploadsForAuthBoundary(previousIdentity)
+ this.activateGfsAuthScope()
this.stopAllStreams()
// Grants are keyed by userId, not by team, so they carry over — but every
// cached org/agents/contexts answer is now about the wrong team. Drop the
diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.ts
index 6a0a205b3..8bf873a08 100644
--- a/desktop-app/src/gfs/upload.test.ts
+++ b/desktop-app/src/gfs/upload.test.ts
@@ -151,6 +151,71 @@ describe('desktop GFS indexed uploader', () => {
}
})
+ it('retries a lifecycle 429 using the transport Retry-After value', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-retry-after-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ await writeFile(filePath, Buffer.from([1]))
+ const session = {
+ uploadId: '13131313-1313-4131-8131-131313131313',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let createCalls = 0
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads')) {
+ createCalls += 1
+ if (createCalls === 1)
+ throw Object.assign(new Error('quota exceeded'), { status: 429, retryAfter: '0' })
+ return { ok: true, data: session } as T
+ }
+ if (method === 'POST' && requestPath(url).endsWith('/complete'))
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* drain */
+ }
+ return { status: 204, text: '' }
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-retry-after',
+ transport,
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(createCalls).toBe(2)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('resumes from sparse status and does not replay a committed part', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-resume-'))
try {
diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.ts
index 22a586986..223156274 100644
--- a/desktop-app/src/gfs/upload.ts
+++ b/desktop-app/src/gfs/upload.ts
@@ -1,7 +1,7 @@
import { createHash, randomUUID } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { stat } from 'node:fs/promises'
-import type { Readable } from 'node:stream'
+import { type Readable, Transform } from 'node:stream'
const GFS_UPLOAD_V2_PRODUCT_MAX_BYTES = 200 * 1024 * 1024
const GFS_UPLOAD_V2_PREFERRED_PART_BYTES = 8 * 1024 * 1024
@@ -11,6 +11,9 @@ const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2
const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
+export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
+export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
+const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
export interface DesktopUploadSession {
uploadId: string
@@ -71,6 +74,7 @@ interface Envelope<T> {
interface UploadPartResponse {
status: number
text: string
+ retryAfter?: string
}
interface UploadCapabilities {
@@ -219,6 +223,8 @@ function retryableError(error: unknown): boolean {
if (error instanceof Error && error.message.startsWith('stale_auth_epoch:')) return false
if (error instanceof Error && error.message.startsWith('upload_part_outcome_unknown:'))
return false
+ if (error instanceof Error && error.message.startsWith('upload_completion_outcome_unknown:'))
+ return false
const status =
error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'
? Number((error as { status: number }).status)
@@ -228,8 +234,150 @@ function retryableError(error: unknown): boolean {
return status === undefined || retryable(status)
}
-function sleep(ms: number): Promise<void> {
- return new Promise(resolve => setTimeout(resolve, ms))
+/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */
+export function parseRetryAfter(
+ value: string | null | undefined,
+ nowMs = Date.now(),
+ capMs = GFS_UPLOAD_RETRY_AFTER_CAP_MS
+): number | undefined {
+ if (value === null || value === undefined) return undefined
+ const trimmed = value.trim()
+ if (!trimmed) return undefined
+ const seconds = Number(trimmed)
+ let delayMs: number
+ if (Number.isFinite(seconds) && seconds >= 0) {
+ delayMs = seconds * 1_000
+ } else {
+ const dateMs = Date.parse(trimmed)
+ if (!Number.isFinite(dateMs)) return undefined
+ delayMs = Math.max(0, dateMs - nowMs)
+ }
+ if (!Number.isFinite(delayMs) || delayMs < 0) return undefined
+ return Math.min(capMs, delayMs)
+}
+
+function retryAfterOf(error: unknown): number | undefined {
+ if (!error || typeof error !== 'object') return undefined
+ const value = error as {
+ retryAfterMs?: unknown
+ retryAfter?: unknown
+ headers?: { get?: (name: string) => string | null }
+ response?: { headers?: { get?: (name: string) => string | null } }
+ bodyText?: unknown
+ }
+ if (typeof value.retryAfterMs === 'number' && Number.isFinite(value.retryAfterMs))
+ return Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, Math.max(0, value.retryAfterMs))
+ const header =
+ typeof value.retryAfter === 'string'
+ ? value.retryAfter
+ : (value.headers?.get?.('retry-after') ?? value.response?.headers?.get?.('retry-after'))
+ const parsedHeader = parseRetryAfter(typeof header === 'string' ? header : undefined)
+ if (parsedHeader !== undefined) return parsedHeader
+ if (typeof value.bodyText === 'string') {
+ try {
+ const parsed = JSON.parse(value.bodyText) as { retryAfterSeconds?: unknown }
+ if (typeof parsed.retryAfterSeconds === 'number' && Number.isFinite(parsed.retryAfterSeconds))
+ return Math.min(
+ GFS_UPLOAD_RETRY_AFTER_CAP_MS,
+ Math.max(0, parsed.retryAfterSeconds * 1_000)
+ )
+ } catch {
+ // Preserve the transport's original error when its body is not JSON.
+ }
+ }
+ return undefined
+}
+
+function ambiguousLifecycleError(error: unknown): boolean {
+ const status =
+ error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'
+ ? Number((error as { status: number }).status)
+ : undefined
+ return status === undefined || status === 408 || (status !== undefined && status >= 500)
+}
+
+function terminalReconciliationError(error: unknown): boolean {
+ if (!(error instanceof Error)) return false
+ return [
+ 'upload status changed during reconciliation',
+ 'resume_part_mismatch:',
+ 'source_changed:',
+ 'upload_completion_outcome_unknown:',
+ 'cannot reconcile ',
+ ].some(prefix => error.message.startsWith(prefix))
+}
+
+interface LifecycleRecovery<T> {
+ value: T
+}
+
+interface LifecycleRetryOptions<T> {
+ signal?: AbortSignal
+ attempts?: number
+ reconcileOnError?: (error: unknown) => boolean
+ reconcile?: (error: unknown) => Promise<LifecycleRecovery<T> | undefined>
+}
+
+/**
+ * Bounded lifecycle retry primitive. It keeps Retry-After handling, abort
+ * propagation, and state-reconciliation hooks identical across Desktop
+ * create/pause/resume/complete/cancel/status requests.
+ */
+async function requestWithLifecycleRetries<T>(
+ input: DesktopUploadInput,
+ operation: () => Promise<T>,
+ options: LifecycleRetryOptions<T> = {}
+): Promise<T> {
+ const attempts = Math.max(1, options.attempts ?? GFS_UPLOAD_RETRY_MAX_ATTEMPTS)
+ let lastError: unknown
+ for (let attempt = 0; attempt < attempts; attempt += 1) {
+ assertAuthEpoch(input)
+ try {
+ return await operation()
+ } catch (error) {
+ lastError = error
+ if (options.signal?.aborted) throw options.signal.reason ?? error
+ const shouldReconcile = options.reconcile
+ ? (options.reconcileOnError?.(error) ?? ambiguousLifecycleError(error))
+ : false
+ if (shouldReconcile) {
+ try {
+ const recovered = await options.reconcile!(error)
+ if (recovered) return recovered.value
+ } catch (reconcileError) {
+ // A transient status/HEAD failure must not suppress the bounded
+ // retry of the original request. Definitive geometry, auth, or
+ // outcome-unknown errors remain terminal and are surfaced.
+ if (terminalReconciliationError(reconcileError)) throw reconcileError
+ }
+ }
+ if (!retryableError(error) || attempt + 1 >= attempts) throw error
+ await sleep(
+ retryAfterOf(error) ??
+ Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ options.signal
+ )
+ }
+ }
+ throw lastError
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) {
+ reject(signal.reason)
+ return
+ }
+ const timer = setTimeout(resolve, ms)
+ signal?.addEventListener(
+ 'abort',
+ () => {
+ clearTimeout(timer)
+ reject(signal.reason)
+ },
+ { once: true }
+ )
+ })
}
interface DesktopUploadPartChecksum {
@@ -349,16 +497,22 @@ async function loadUploadStatus(
partCount?: number
},
signal?: AbortSignal,
- timeoutMs = UPLOAD_TIMEOUT_MS
+ timeoutMs = UPLOAD_TIMEOUT_MS,
+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS
): Promise<DesktopUploadStatus> {
assertAuthEpoch(input)
- await input.transport.requestJson<Envelope<undefined>>(
- 'HEAD',
- joinUrl(
- input.baseUrl,
- uploadPath(expected.drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}`)
- ),
- { token: input.token, timeoutMs, signal }
+ await requestWithLifecycleRetries(
+ input,
+ () =>
+ input.transport.requestJson<Envelope<undefined>>(
+ 'HEAD',
+ joinUrl(
+ input.baseUrl,
+ uploadPath(expected.drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}`)
+ ),
+ { token: input.token, timeoutMs, signal }
+ ),
+ { signal, attempts: retryAttempts }
)
let cursor: string | undefined
@@ -369,17 +523,22 @@ async function loadUploadStatus(
const query = new URLSearchParams({ limit: '256' })
if (cursor) query.set('cursor', cursor)
const status = unwrap<DesktopUploadStatus>(
- await input.transport.requestJson<Envelope<DesktopUploadStatus>>(
- 'GET',
- joinUrl(
- input.baseUrl,
- uploadPath(
- expected.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}/status`,
- query
- )
- ),
- { token: input.token, timeoutMs, signal }
+ await requestWithLifecycleRetries(
+ input,
+ () =>
+ input.transport.requestJson<Envelope<DesktopUploadStatus>>(
+ 'GET',
+ joinUrl(
+ input.baseUrl,
+ uploadPath(
+ expected.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}/status`,
+ query
+ )
+ ),
+ { token: input.token, timeoutMs, signal }
+ ),
+ { signal, attempts: retryAttempts }
)
)
session ??= status.session
@@ -440,14 +599,18 @@ async function reconcileAmbiguousPart(
partCount: session.partCount,
},
signal,
- GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,
+ 1
)
} catch (error) {
if (signal?.aborted) throw signal.reason ?? error
lastError = error
if (!retryableError(error) || attempt + 1 >= GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)
throw partOutcomeUnknown(partNumber, error)
- await sleep(250 * 2 ** attempt)
+ await sleep(
+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ signal
+ )
continue
}
@@ -485,6 +648,7 @@ async function putPart(
start: number,
length: number,
signal?: AbortSignal,
+ onByteProgress?: (loadedBytes: number) => void,
onRetryableFailure?: () => void,
onPartSuccess?: () => void
): Promise<void> {
@@ -494,11 +658,21 @@ async function putPart(
await assertSourceIdentity(input)
const checksum = await checksumPart(input.filePath, start, start + length - 1)
await assertSourceIdentity(input)
- const body = createReadStream(input.filePath, {
+ const source = createReadStream(input.filePath, {
start,
end: start + length - 1,
highWaterMark: 1024 * 1024,
})
+ let loadedBytes = 0
+ const body = new Transform({
+ transform(chunk, _encoding, callback) {
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+ loadedBytes += buffer.length
+ onByteProgress?.(Math.min(length, loadedBytes))
+ callback(null, buffer)
+ },
+ })
+ source.pipe(body)
let response: UploadPartResponse
try {
assertAuthEpoch(input)
@@ -541,7 +715,10 @@ async function putPart(
return
}
if (attempt >= 2) throw error
- await sleep(250 * 2 ** attempt)
+ await sleep(
+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ signal
+ )
attempt += 1
continue
}
@@ -569,7 +746,10 @@ async function putPart(
}
}
if (attempt >= 2) throw uploadPartError(response.status, response.text)
- await sleep(250 * 2 ** attempt)
+ await sleep(
+ retryAfterOf(response) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
+ signal
+ )
attempt += 1
}
}
@@ -595,8 +775,10 @@ async function runParts(
let retryableFailures = 0
let consecutiveRetryableFailures = 0
let downgraded = false
+ const inFlightLoaded = new Map<number, number>()
const advertisedConcurrency = Number(input.advertisedConcurrency)
const concurrency = Math.min(
+ GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,
Number.isSafeInteger(advertisedConcurrency) && advertisedConcurrency > 0
? advertisedConcurrency
: GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,
@@ -621,6 +803,21 @@ async function runParts(
const registerPartSuccess = (): void => {
consecutiveRetryableFailures = 0
}
+ const calculateUploadedBytes = (): number => {
+ const committedBytes = [...completedParts].reduce(
+ (sum, number) =>
+ sum + Math.min(session.partBytes, session.expectedBytes - number * session.partBytes),
+ 0
+ )
+ const inFlightBytes = [...inFlightLoaded.entries()].reduce(
+ (sum, [number, loaded]) => (completedParts.has(number) ? sum : sum + loaded),
+ 0
+ )
+ const observed = Math.min(session.expectedBytes, committedBytes + inFlightBytes)
+ // Part bytes are committed before the final resource publication. Do not
+ // expose a visually complete bar until the complete receipt is durable.
+ return Math.min(Math.max(0, session.expectedBytes - 1), observed)
+ }
const execute = async (work: typeof parts, workers: number): Promise<void> => {
let next = 0
const runWorker = async (workerIndex: number): Promise<void> => {
@@ -641,19 +838,24 @@ async function runParts(
part.offsetBytes,
part.lengthBytes,
signal,
+ loaded => {
+ inFlightLoaded.set(
+ part.partNumber,
+ Math.max(inFlightLoaded.get(part.partNumber) ?? 0, loaded)
+ )
+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)
+ },
registerRetryableFailure,
registerPartSuccess
)
completedParts.add(part.partNumber)
+ inFlightLoaded.delete(part.partNumber)
failed.delete(part.partNumber)
retryableFailed.delete(part.partNumber)
- const completedBytes = [...completedParts].reduce(
- (sum, number) =>
- sum + Math.min(session.partBytes, session.expectedBytes - number * session.partBytes),
- 0
- )
- input.onProgress?.(completedBytes, session.expectedBytes)
+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)
} catch (error) {
+ inFlightLoaded.delete(part.partNumber)
+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)
if (shouldPause()) return
if (
signal?.aborted ||
@@ -684,6 +886,11 @@ function uploadPartError(status: number, text: string): Error & { status: number
return error
}
+function errorCodeOf(error: unknown): string | undefined {
+ const value = error as { code?: unknown } | null
+ return value && typeof value.code === 'string' ? value.code : undefined
+}
+
export class DesktopGfsUploadJob {
private readonly abortController = new AbortController()
private readonly authBoundaryController = new AbortController()
@@ -746,23 +953,31 @@ export class DesktopGfsUploadJob {
throw new Error(`cannot pause upload in ${session.state}`)
this.pauseRequested = true
this.assertAuthEpoch()
- const paused = unwrap<DesktopUploadSession>(
- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
- 'POST',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/pause`
+ const paused = await requestWithLifecycleRetries(
+ this.input,
+ async () =>
+ unwrap<DesktopUploadSession>(
+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
+ 'POST',
+ joinUrl(
+ this.input.baseUrl,
+ uploadPath(
+ this.input.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/pause`
+ )
+ ),
+ {
+ token: this.input.token,
+ body: {},
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.authBoundaryController.signal,
+ }
)
),
- {
- token: this.input.token,
- body: {},
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.authBoundaryController.signal,
- }
- )
+ {
+ signal: this.authBoundaryController.signal,
+ reconcile: () => this.reconcileMutation(session, 'pause'),
+ }
)
this.session = paused
this.state = 'paused'
@@ -781,23 +996,31 @@ export class DesktopGfsUploadJob {
if (session.state !== 'paused') return this.start()
this.pauseRequested = false
this.assertAuthEpoch()
- const resumed = unwrap<DesktopUploadSession>(
- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
- 'POST',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/resume`
+ const resumed = await requestWithLifecycleRetries(
+ this.input,
+ async () =>
+ unwrap<DesktopUploadSession>(
+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
+ 'POST',
+ joinUrl(
+ this.input.baseUrl,
+ uploadPath(
+ this.input.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/resume`
+ )
+ ),
+ {
+ token: this.input.token,
+ body: {},
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.authBoundaryController.signal,
+ }
)
),
- {
- token: this.input.token,
- body: {},
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.authBoundaryController.signal,
- }
- )
+ {
+ signal: this.authBoundaryController.signal,
+ reconcile: () => this.reconcileMutation(session, 'resume'),
+ }
)
this.session = resumed
this.state = 'uploading'
@@ -818,65 +1041,68 @@ export class DesktopGfsUploadJob {
this.abortController.abort(new Error('GFS upload canceled'))
const session = this.session
if (session && !['completed', 'aborted'].includes(session.state)) {
+ let reconciledCompleted = false
try {
- this.assertAuthEpoch()
- await this.input.transport.requestJson(
- 'DELETE',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}`
- )
- ),
+ await requestWithLifecycleRetries(
+ this.input,
+ () =>
+ this.input.transport
+ .requestJson(
+ 'DELETE',
+ joinUrl(
+ this.input.baseUrl,
+ uploadPath(
+ this.input.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}`
+ )
+ ),
+ {
+ token: this.input.token,
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.authBoundaryController.signal,
+ }
+ )
+ .then(() => undefined),
{
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
signal: this.authBoundaryController.signal,
+ reconcileOnError: error =>
+ ambiguousLifecycleError(error) ||
+ Boolean(
+ error && typeof error === 'object' && (error as { status?: unknown }).status === 409
+ ) ||
+ errorCodeOf(error) === 'upload_finalizing',
+ reconcile: async error => {
+ const observed = await this.observeSession(
+ session,
+ GFS_UPLOAD_RETRY_MAX_ATTEMPTS,
+ this.authBoundaryController.signal
+ )
+ this.session = observed
+ if (observed.state === 'completed') {
+ reconciledCompleted = true
+ return { value: undefined }
+ }
+ if (observed.state === 'aborted') return { value: undefined }
+ if (observed.state === 'finalizing') throw error
+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined
+ throw new Error(`cannot reconcile cancel upload in ${observed.state}`)
+ },
}
)
} catch (error) {
- const status =
- error &&
- typeof error === 'object' &&
- typeof (error as { status?: unknown }).status === 'number'
- ? Number((error as { status: number }).status)
- : undefined
- if (status === 409) {
- this.assertAuthEpoch()
- const observed = unwrap<DesktopUploadStatus>(
- await this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(
- 'GET',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/status`,
- new URLSearchParams({ limit: '256' })
- )
- ),
- {
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.authBoundaryController.signal,
- }
- )
- )
- this.session = observed.session
- if (observed.session.state === 'completed') {
- this.canceled = false
- this.state = 'completed'
- this.uploadedBytes = observed.session.expectedBytes
- this.emit()
- await this.input.onClearPersisted?.(session.uploadId)
- return
- }
- }
this.canceled = false
this.state = 'failed'
this.emit()
throw error
}
+ if (reconciledCompleted) {
+ this.canceled = false
+ this.state = 'completed'
+ this.uploadedBytes = this.session?.expectedBytes ?? session.expectedBytes
+ this.emit()
+ await this.input.onClearPersisted?.(session.uploadId)
+ return
+ }
}
await this.runPromise?.catch(() => undefined)
this.state = 'aborted'
@@ -939,6 +1165,70 @@ export class DesktopGfsUploadJob {
}
}
+ private async observeSession(
+ session: DesktopUploadSession,
+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS,
+ signal: AbortSignal | undefined = this.authBoundaryController.signal
+ ): Promise<DesktopUploadSession> {
+ return (
+ await loadUploadStatus(
+ this.input,
+ session.uploadId,
+ {
+ drive: session.drive,
+ expectedBytes: session.expectedBytes,
+ partBytes: session.partBytes,
+ partCount: session.partCount,
+ },
+ signal,
+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,
+ retryAttempts
+ )
+ ).session
+ }
+
+ private async reconcileMutation(
+ session: DesktopUploadSession,
+ action: 'pause' | 'resume'
+ ): Promise<LifecycleRecovery<DesktopUploadSession> | undefined> {
+ const observed = await this.observeSession(session)
+ if (
+ (action === 'pause' && observed.state === 'paused') ||
+ (action === 'resume' && ['initiated', 'uploading', 'completed'].includes(observed.state))
+ ) {
+ return { value: observed }
+ }
+ if (
+ (action === 'pause' && ['initiated', 'uploading'].includes(observed.state)) ||
+ (action === 'resume' && observed.state === 'paused')
+ ) {
+ return undefined
+ }
+ throw new Error(`cannot reconcile ${action} upload in ${observed.state}`)
+ }
+
+ private async reconcileCompletion(): Promise<
+ LifecycleRecovery<DesktopUploadSession> | undefined
+ > {
+ const session = this.session
+ if (!session) throw new Error('GFS upload session is unavailable during completion')
+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {
+ const observed = await this.observeSession(session, 1)
+ if (observed.state === 'completed') return { value: observed }
+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined
+ if (observed.state !== 'finalizing')
+ throw new Error(`cannot reconcile upload completion in ${observed.state}`)
+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)
+ await sleep(
+ GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt,
+ this.authBoundaryController.signal
+ )
+ }
+ throw new Error(
+ 'upload_completion_outcome_unknown: finalization is still in progress; resume the persisted session'
+ )
+ }
+
private async prepare(): Promise<{
session: DesktopUploadSession
committed: Set<number>
@@ -959,14 +1249,19 @@ export class DesktopGfsUploadJob {
// receipts use the { ok, data, error } envelope; accepting an envelope
// here would hide drift between Desktop and the writer's public contract.
capabilities = parseUploadCapabilities(
- await this.input.transport.requestJson<UploadCapabilities>(
- 'GET',
- joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/capabilities')),
- {
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.abortController.signal,
- }
+ await requestWithLifecycleRetries(
+ this.input,
+ () =>
+ this.input.transport.requestJson<UploadCapabilities>(
+ 'GET',
+ joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/capabilities')),
+ {
+ token: this.input.token,
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.abortController.signal,
+ }
+ ),
+ { signal: this.abortController.signal }
)
)
} catch (error) {
@@ -979,8 +1274,14 @@ export class DesktopGfsUploadJob {
throw new DesktopUploadCapabilityError(
'Resumable GFS uploads are not enabled on this writer.'
)
- if (typeof resumable.maxFileBytes === 'number' && file.size > resumable.maxFileBytes)
- throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)
+ if (resumable.maxFileBytes !== undefined) {
+ if (!Number.isSafeInteger(resumable.maxFileBytes) || resumable.maxFileBytes < 0)
+ throw new DesktopUploadCapabilityError(
+ 'GFS resumable capabilities advertised an invalid file ceiling'
+ )
+ if (file.size > resumable.maxFileBytes)
+ throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)
+ }
this.input.advertisedConcurrency = resumable.maxConcurrentPartsPerSession
this.input.fallbackConcurrency = resumable.fallbackConcurrency
const resumeId = this.session?.uploadId ?? this.input.resumeUploadId
@@ -1001,24 +1302,30 @@ export class DesktopGfsUploadJob {
const target = this.input.operation === 'create' ? this.input.parentRid : this.input.resourceRid
if (!target) throw new Error(`${this.input.operation} upload target is required`)
this.assertAuthEpoch()
+ const createBody = {
+ operation: this.input.operation,
+ drive,
+ sizeBytes: file.size,
+ idempotencyKey: randomUUID(),
+ ...(this.input.operation === 'create'
+ ? { parentRid: target, name: this.input.name }
+ : { resourceRid: target, ifMatch: this.input.ifMatch }),
+ }
const created = unwrap<DesktopUploadSession>(
- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
- 'POST',
- joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/uploads')),
- {
- token: this.input.token,
- body: {
- operation: this.input.operation,
- drive,
- sizeBytes: file.size,
- idempotencyKey: randomUUID(),
- ...(this.input.operation === 'create'
- ? { parentRid: target, name: this.input.name }
- : { resourceRid: target, ifMatch: this.input.ifMatch }),
- },
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.abortController.signal,
- }
+ await requestWithLifecycleRetries(
+ this.input,
+ () =>
+ this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
+ 'POST',
+ joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/uploads')),
+ {
+ token: this.input.token,
+ body: createBody,
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.abortController.signal,
+ }
+ ),
+ { signal: this.abortController.signal }
)
)
if (created.drive !== drive) throw new Error('upload_drive_mismatch: writer changed drive')
@@ -1080,21 +1387,26 @@ export class DesktopGfsUploadJob {
this.state = 'paused'
this.assertAuthEpoch()
const status = unwrap<DesktopUploadStatus>(
- await this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(
- 'GET',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,
- new URLSearchParams({ limit: '256' })
- )
- ),
- {
- token: this.input.token,
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.authBoundaryController.signal,
- }
+ await requestWithLifecycleRetries(
+ this.input,
+ () =>
+ this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(
+ 'GET',
+ joinUrl(
+ this.input.baseUrl,
+ uploadPath(
+ this.input.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,
+ new URLSearchParams({ limit: '256' })
+ )
+ ),
+ {
+ token: this.input.token,
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.authBoundaryController.signal,
+ }
+ ),
+ { signal: this.authBoundaryController.signal }
)
)
this.session = status.session
@@ -1108,23 +1420,33 @@ export class DesktopGfsUploadJob {
this.assertAuthEpoch()
this.state = 'finalizing'
this.emit()
- const completed = unwrap<DesktopUploadSession>(
- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
- 'POST',
- joinUrl(
- this.input.baseUrl,
- uploadPath(
- this.input.drive,
- `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`
+ const completed = await requestWithLifecycleRetries<DesktopUploadSession>(
+ this.input,
+ async () =>
+ unwrap<DesktopUploadSession>(
+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(
+ 'POST',
+ joinUrl(
+ this.input.baseUrl,
+ uploadPath(
+ this.input.drive,
+ `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`
+ )
+ ),
+ {
+ token: this.input.token,
+ body: {},
+ timeoutMs: UPLOAD_TIMEOUT_MS,
+ signal: this.abortController.signal,
+ }
)
),
- {
- token: this.input.token,
- body: {},
- timeoutMs: UPLOAD_TIMEOUT_MS,
- signal: this.abortController.signal,
- }
- )
+ {
+ signal: this.abortController.signal,
+ reconcileOnError: error =>
+ ambiguousLifecycleError(error) || errorCodeOf(error) === 'upload_finalizing',
+ reconcile: () => this.reconcileCompletion(),
+ }
)
if (completed.drive !== canonicalDrive(this.input.drive))
throw new Error('upload_drive_mismatch: completion changed drive')
diff --git a/desktop-app/src/httpClient.ts b/desktop-app/src/httpClient.ts
index 5ab838154..3013b5fae 100644
--- a/desktop-app/src/httpClient.ts
+++ b/desktop-app/src/httpClient.ts
@@ -3,11 +3,13 @@ import { config } from './config.js'
export class ApiError extends Error {
status: number
bodyText: string
+ retryAfter?: string
- constructor(message: string, status: number, bodyText: string) {
+ constructor(message: string, status: number, bodyText: string, retryAfter?: string | null) {
super(message)
this.status = status
this.bodyText = bodyText
+ if (retryAfter) this.retryAfter = retryAfter
}
}
@@ -125,7 +127,12 @@ export async function requestJson<T>(
} catch {
// Keep raw text as message.
}
- throw new ApiError(`${response.status} ${response.statusText}: ${msg}`, response.status, raw)
+ throw new ApiError(
+ `${response.status} ${response.statusText}: ${msg}`,
+ response.status,
+ raw,
+ response.headers.get('retry-after')
+ )
}
if (!raw) return {} as T
diff --git a/desktop-app/test/httpClient.test.ts b/desktop-app/test/httpClient.test.ts
index 611cb47b0..d1cd695dd 100644
--- a/desktop-app/test/httpClient.test.ts
+++ b/desktop-app/test/httpClient.test.ts
@@ -16,12 +16,13 @@ function okResponse(body?: unknown): Response {
} as unknown as Response
}
-function errorResponse(status: number, body: string): Response {
+function errorResponse(status: number, body: string, headers?: Record<string, string>): Response {
return {
ok: false,
status,
statusText: 'Error',
text: () => Promise.resolve(body),
+ headers: new Headers(headers),
} as unknown as Response
}
@@ -91,4 +92,16 @@ describe('requestJson — transient retry', () => {
expect(fetchSpy).toHaveBeenCalledTimes(1)
})
+
+ it('preserves Retry-After on HTTP errors for bounded upload lifecycle retries', async () => {
+ const fetchSpy = vi.mocked(global.fetch)
+ fetchSpy.mockResolvedValueOnce(
+ errorResponse(429, '{"error":"quota_exceeded"}', { 'retry-after': '7' })
+ )
+
+ await expect(requestJson('POST', 'http://localhost/upload')).rejects.toMatchObject({
+ status: 429,
+ retryAfter: '7',
+ })
+ })
})
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index 6447f3a0b..c589ae108 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.68",
+ "version": "0.1.69",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.68",
+ "version": "0.1.69",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index f21f91fb9..b98e1c008 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.68",
+ "version": "0.1.69",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/routes/gfs.ts b/external-rest-api/src/routes/gfs.ts
index cd1c53063..08779f96e 100644
--- a/external-rest-api/src/routes/gfs.ts
+++ b/external-rest-api/src/routes/gfs.ts
@@ -33,6 +33,7 @@ const STREAM_HEADERS = [
'content-length',
'content-disposition',
'cache-control',
+ 'location',
'upload-offset',
'upload-length',
'upload-part-bytes',
@@ -44,14 +45,19 @@ const STREAM_HEADERS = [
'upload-part-offset',
'upload-part-length',
'upload-checksum',
+ 'retry-after',
+ 'x-ratelimit-limit',
+ 'x-ratelimit-remaining',
]
function forwardControlApiError(error: unknown, res: Response, next: NextFunction): void {
if (error instanceof ControlApiError && PROPAGATED.has(error.status)) {
const body =
error.body && typeof error.body === 'object' ? error.body : { error: String(error.message) }
- const retryAfter = error.headers?.get('retry-after')
- if (retryAfter) res.setHeader('Retry-After', retryAfter)
+ for (const header of STREAM_HEADERS) {
+ const value = error.headers?.get(header)
+ if (value) res.setHeader(header, value)
+ }
res.status(error.status).json(body)
return
}
diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.ts
index 3b5f06e1a..5e8160092 100644
--- a/gfs-controller/src/api/serve.test.ts
+++ b/gfs-controller/src/api/serve.test.ts
@@ -708,6 +708,27 @@ describe("GfsServingHandler — write routes (governed mutation)", () => {
expect(calls).toHaveLength(0); // never reached the write service
});
+ it.each([-1, 1.5, "2"])(
+ "400 path_invalid for an invalid If-Match value before authorization: %j",
+ async (ifMatch) => {
+ const res = new FakeRes();
+ const { d, calls, authzOps } = writeDeps();
+ await run(
+ d,
+ reqBody(`/v1/resources/${RID}/content`, {
+ method: "PUT",
+ auth: "Bearer t",
+ body: { content: "new", ifMatch },
+ }),
+ res
+ );
+ expect(res.statusCode).toBe(400);
+ expect((res.json as { error: { code: string } }).error.code).toBe("path_invalid");
+ expect(authzOps).toEqual([]);
+ expect(calls).toHaveLength(0);
+ }
+ );
+
it("a NON-agent (user) replace WITHOUT If-Match is allowed (agent invariant only)", async () => {
const res = new FakeRes();
const { d, calls } = writeDeps({ verifyToken: () => USER_WRITE_CLAIMS });