Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): close resumable upload review gaps
Alfredo Lopez5bac17aa9c52
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.tsindex 0ab7bd6af..8a9e4aea4 100644--- a/control-api/src/routes/external/gfs.ts+++ b/control-api/src/routes/external/gfs.ts@@ -41,6 +41,26 @@ import { GFS_DEFAULT_DRIVE, parseRequestedGfsScopes } from '../gfs/token.js'// rejected here, not forwarded to gfsc as an unbounded request.const MAX_GFS_URI_LEN = 2048const ACCESSIBLE_RESOURCE_DEFAULT_LIMIT = 100+const GFS_UPLOAD_RESPONSE_HEADERS = [+ 'content-type',+ 'content-length',+ 'cache-control',+ 'location',+ 'upload-offset',+ 'upload-length',+ 'upload-part-bytes',+ 'upload-part-count',+ 'upload-active-parts',+ 'upload-state',+ 'upload-expires',+ 'upload-part-number',+ 'upload-part-offset',+ 'upload-part-length',+ 'upload-checksum',+ 'retry-after',+ 'x-ratelimit-limit',+ 'x-ratelimit-remaining',+] as consttype ExternalGfsRequest = ExternalAuthedRequest & {gfsSubjectKeys?: string[]@@ -925,31 +945,15 @@ async function proxyUploadToGfsc('gfs external upload upstream error')res.status(upstream.status)- const contentType = upstream.headers.get('content-type')- if (contentType) res.setHeader('content-type', contentType)+ for (const name of GFS_UPLOAD_RESPONSE_HEADERS) {+ const value = upstream.headers.get(name)+ if (value) res.setHeader(name, value)+ }res.send(body)return}res.status(upstream.status)- for (const name of [- 'content-type',- 'content-length',- 'cache-control',- 'upload-offset',- 'upload-length',- 'upload-part-bytes',- 'upload-part-count',- 'upload-active-parts',- 'upload-state',- 'upload-expires',- 'upload-part-number',- 'upload-part-offset',- 'upload-part-length',- 'upload-checksum',- 'retry-after',- 'x-ratelimit-limit',- 'x-ratelimit-remaining',- ]) {+ for (const name of GFS_UPLOAD_RESPONSE_HEADERS) {const value = upstream.headers.get(name)if (value) res.setHeader(name, value)}diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.tsindex 895fa32dc..adc573f95 100644--- a/control-api/src/routes/gfs/proxy.ts+++ b/control-api/src/routes/gfs/proxy.ts@@ -11,7 +11,24 @@ import { type UiAuthedRequest, requireAuthForControlUI } from '../../middleware/import { rootLogger } from '../../observability/logger.js'const DEFAULT_DRIVE = 'main'-const PASSTHROUGH_RESPONSE_HEADERS = ['content-type', 'content-disposition', 'content-length']+const PASSTHROUGH_RESPONSE_HEADERS = [+ 'content-type',+ 'content-disposition',+ 'content-length',+ 'location',+ 'upload-offset',+ 'upload-length',+ 'upload-part-bytes',+ 'upload-part-count',+ 'upload-active-parts',+ 'upload-state',+ 'upload-expires',+ 'upload-part-number',+ 'upload-part-offset',+ 'upload-part-length',+ 'upload-checksum',+ 'retry-after',+]const UPLOAD_PART_PATH = /^\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+(?:\?|$)/iconst UPLOAD_PATH = /^\/v1\/(?:capabilities|uploads)(?:\/|\?|$)/iconst STREAM_REQUEST_HEADERS = [@@ -137,6 +154,12 @@ export function registerGfsProxyRoute(router: Router): void {res.status(upstreamRes.status)const contentType = upstreamRes.headers.get('content-type')if (contentType) res.setHeader('content-type', contentType)+ const retryAfter = upstreamRes.headers.get('retry-after')+ if (retryAfter) res.setHeader('retry-after', retryAfter)+ for (const header of PASSTHROUGH_RESPONSE_HEADERS) {+ const value = upstreamRes.headers.get(header)+ if (value) res.setHeader(header, value)+ }res.send(errorBody)return}diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex bc9309875..d5d8ce30c 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -33,6 +33,8 @@ export function currentWindowStartMs(nowMs = Date.now()): number {return Math.floor(nowMs / WINDOW_MS) * WINDOW_MS}+type RateLimitQuery = (text: string, values?: unknown[]) => Promise<{ rows: unknown[] }>+/*** Atomically increment the (bucketKey, windowStart) counter, return the new* count plus whether it's still under `maxPerMinute`.@@ -47,6 +49,28 @@ export async function checkAndIncrement(maxPerMinute: number,nowMs = Date.now(),cost = 1+): Promise<RateLimitCheck> {+ return checkAndIncrementWithQuery(+ (text, values) => pool.query(text, values),+ bucketKey,+ maxPerMinute,+ nowMs,+ cost+ )+}++/**+ * Same atomic limiter operation against an explicitly supplied query+ * function. Production callers use the process-wide pool above; the narrow+ * seam lets the real-Postgres integration suite exercise this exact SQL+ * against an isolated database without changing runtime ownership.+ */+export async function checkAndIncrementWithQuery(+ query: RateLimitQuery,+ bucketKey: string,+ maxPerMinute: number,+ nowMs = Date.now(),+ cost = 1): Promise<RateLimitCheck> {if (!Number.isSafeInteger(cost) || cost < 1) throw new Error('rate limit cost must be positive')const windowStartMs = currentWindowStartMs(nowMs)@@ -54,7 +78,7 @@ export async function checkAndIncrement(let result: { rows: unknown[] } | null | undefinedtry {- result = await pool.query(+ result = await query(`INSERT INTO rate_limit_buckets (bucket_key, window_start_ms, count)VALUES ($1, $2, $3)ON CONFLICT (bucket_key, window_start_ms) DO UPDATEdiff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.tsindex 60d84dc67..06771f2f9 100644--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts@@ -1,12 +1,12 @@import { afterAll, beforeAll, describe, expect, it } from 'vitest'import { createHash, randomBytes, randomUUID } from 'node:crypto'-import { mkdtemp, rm } from 'node:fs/promises'+import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import { Readable } from 'node:stream'import { Pool } from 'pg'import type { GfsUploadConfig } from '../../gfs-controller/src/config.js'-import { PgTransactor } from '../../gfs-controller/src/db/writeStore.js'+import { CommitOutcomeUnknownError, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'import type {TransactionOptions,Transactor,@@ -68,6 +68,32 @@ class HoldBeforeCommitTransactor implements Transactor {}}+/**+ * Simulates a network/process failure after PostgreSQL has committed. The+ * upload service must reconcile the durable row instead of replaying a+ * state-changing operation blindly.+ */+class ThrowAfterCommitTransactor implements Transactor {+ private transactionCount = 0++ constructor(+ private readonly delegate: Transactor,+ private readonly failOnTransaction: number+ ) {}++ async transaction<T>(+ fn: (client: TxClient) => Promise<T>,+ options?: TransactionOptions+ ): Promise<T> {+ this.transactionCount += 1+ const result = await this.delegate.transaction(fn, options)+ if (this.transactionCount === this.failOnTransaction) {+ throw new CommitOutcomeUnknownError(new Error('synthetic post-commit response loss'))+ }+ return result+ }+}+const CONFIG: GfsUploadConfig = {productMaxFileBytes: 209715200,protocolMaxFileBytes: 1073741824,@@ -222,6 +248,241 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {expect(rows.rows).toEqual(expect.arrayContaining([{ state: 'committed', count: 2 }]))})+ it('serializes concurrent indexed part reservations without losing either commit', async () => {+ const bytes = [+ Buffer.alloc(CONFIG.preferredPartBytes, 0x31),+ Buffer.alloc(CONFIG.preferredPartBytes, 0x32),+ ]+ const created = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '7'.repeat(32),+ name: 'concurrent-parts.bin',+ sizeBytes: bytes.length * CONFIG.preferredPartBytes,+ idempotencyKey: randomUUID(),+ })+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ const results = await Promise.all(+ bytes.map((value, partNumber) =>+ uploads.putPart(+ created.session.uploadId,+ partGeometry(+ { expectedBytes: value.length * bytes.length, partBytes: CONFIG.preferredPartBytes },+ partNumber+ ),+ checksum(value),+ Readable.from([value]) as never,+ principal+ )+ )+ )+ expect(results.map(result => result.part.state)).toEqual(['committed', 'committed'])+ const count = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,+ [created.session.uploadId]+ )+ expect(count.rows[0]?.count).toBe(2)+ })++ it('completes non-zero parts and replays complete idempotently', async () => {+ const bytes0 = Buffer.alloc(CONFIG.preferredPartBytes, 0x61)+ const bytes1 = Buffer.alloc(CONFIG.preferredPartBytes, 0x62)+ const uploadsWithFinalizer = new GfsUploadSessionService({+ db: pool,+ tx: new PgTransactor(pool),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ finalize: async session => ({+ resourceId: session.uploadId,+ version: 1,+ sha256: checksum(Buffer.concat([bytes0, bytes1])),+ }),+ })+ const created = await uploadsWithFinalizer.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: 'f'.repeat(32),+ name: 'complete-non-zero.bin',+ sizeBytes: bytes0.length + bytes1.length,+ idempotencyKey: randomUUID(),+ })+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ for (const [partNumber, bytes] of [bytes0, bytes1].entries()) {+ await uploadsWithFinalizer.putPart(+ created.session.uploadId,+ partGeometry(+ { expectedBytes: bytes0.length + bytes1.length, partBytes: CONFIG.preferredPartBytes },+ partNumber+ ),+ checksum(bytes),+ Readable.from([bytes]) as never,+ principal+ )+ }++ const completed = await uploadsWithFinalizer.complete(created.session.uploadId, principal)+ expect(completed).toMatchObject({+ uploadId: created.session.uploadId,+ state: 'completed',+ committedBytes: bytes0.length + bytes1.length,+ resultVersion: 1,+ })+ await expect(+ uploadsWithFinalizer.complete(created.session.uploadId, principal)+ ).resolves.toEqual(completed)+ })++ it('reconciles an ambiguous committed part response without duplicating the part', async () => {+ const base = new GfsUploadSessionService({+ db: pool,+ tx: new PgTransactor(pool),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ })+ const created = await base.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '1'.repeat(32),+ name: 'ambiguous-part.bin',+ sizeBytes: CONFIG.preferredPartBytes,+ idempotencyKey: randomUUID(),+ })+ const uploadsWithUnknownCommit = new GfsUploadSessionService({+ db: pool,+ // The reservation transaction commits first; the second transaction is+ // the indexed-part commit whose response is intentionally lost.+ tx: new ThrowAfterCommitTransactor(new PgTransactor(pool), 2),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ })+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x73)+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ const result = await uploadsWithUnknownCommit.putPart(+ created.session.uploadId,+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),+ checksum(bytes),+ Readable.from([bytes]) as never,+ principal+ )+ expect(result.part.state).toBe('committed')+ const count = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,+ [created.session.uploadId]+ )+ expect(count.rows[0]?.count).toBe(1)+ })++ it('reconciles an ambiguous complete response from the committed session row', async () => {+ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x84)+ const base = new GfsUploadSessionService({+ db: pool,+ tx: new PgTransactor(pool),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ })+ const created = await base.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '2'.repeat(32),+ name: 'ambiguous-complete.bin',+ sizeBytes: bytes.length,+ idempotencyKey: randomUUID(),+ })+ const principal = { drive, ownerSubject: owner, primarySubject: owner }+ await base.putPart(+ created.session.uploadId,+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),+ checksum(bytes),+ Readable.from([bytes]) as never,+ principal+ )+ const uploadsWithFinalizer = new GfsUploadSessionService({+ db: pool,+ tx: new ThrowAfterCommitTransactor(new PgTransactor(pool), 2),+ blobs: { availableBytes: async () => 10n ** 12n } as never,+ storageMountPath: tempRoot,+ config: CONFIG,+ finalize: async session => ({+ resourceId: session.uploadId,+ version: 2,+ sha256: checksum(bytes),+ }),+ })+ await expect(+ uploadsWithFinalizer.complete(created.session.uploadId, principal)+ ).resolves.toMatchObject({+ state: 'completed',+ resultVersion: 2,+ })+ })++ it('fences and cleans a stale reserved lease while its row lock is held', async () => {+ const created = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '3'.repeat(32),+ name: 'stale-reserved.bin',+ sizeBytes: CONFIG.preferredPartBytes,+ idempotencyKey: randomUUID(),+ })+ const partsDir = join(tempRoot, '.uploads', created.session.uploadId, 'parts')+ await mkdir(partsDir, { recursive: true })+ const staleSuffix = `part.tmp-${randomUUID()}`+ const stagingPath = join(partsDir, `0.${staleSuffix}`)+ const finalPath = join(partsDir, '0.part')+ await writeFile(stagingPath, Buffer.from('stale-temp'))+ await writeFile(finalPath, Buffer.from('stale-final'))+ await pool.query(+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = 1 WHERE upload_id = $1`,+ [created.session.uploadId]+ )+ await pool.query(+ `INSERT INTO gfs_upload_parts+ (upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch, lease_started_at)+ VALUES ($1, 0, 0, $2, $3, 'reserved', $4, 1, now() - interval '1 hour')`,+ [+ created.session.uploadId,+ CONFIG.preferredPartBytes,+ checksum(Buffer.from('stale')),+ stagingPath,+ ]+ )++ const result = await uploads.reconcile()++ expect(result.staleParts).toBeGreaterThanOrEqual(1)+ expect(+ await pool.query('SELECT state FROM gfs_upload_parts WHERE upload_id = $1', [+ created.session.uploadId,+ ])+ ).toMatchObject({+ rows: [{ state: 'failed' }],+ })+ expect(+ (+ await pool.query('SELECT active_part_count FROM gfs_upload_sessions WHERE upload_id = $1', [+ created.session.uploadId,+ ])+ ).rows[0]?.active_part_count+ ).toBe(0)+ await expect(stat(stagingPath)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(stat(finalPath)).rejects.toMatchObject({ code: 'ENOENT' })+ })+it('does not disclose a session to another owner and makes cancel terminal', async () => {const created = await uploads.create({drive,@@ -323,6 +584,63 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {expect(parts.rows[0]?.count).toBe(0)})+ it('reconciles expired sessions and detects a committed part whose file is missing', async () => {+ const expired = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '9'.repeat(32),+ name: 'expired-reconcile.bin',+ sizeBytes: CONFIG.preferredPartBytes,+ idempotencyKey: randomUUID(),+ })+ await pool.query(+ `UPDATE gfs_upload_sessions SET expires_at = now() - interval '1 second' WHERE upload_id = $1`,+ [expired.session.uploadId]+ )++ const expiredResult = await uploads.reconcile()+ expect(expiredResult.expiredSessions).toBeGreaterThanOrEqual(1)+ await expect(+ pool.query(`SELECT state FROM gfs_upload_sessions WHERE upload_id = $1`, [+ expired.session.uploadId,+ ])+ ).resolves.toMatchObject({ rows: [{ state: 'expired' }] })++ const bytes = Buffer.alloc(CONFIG.preferredPartBytes, 0x5a)+ const committed = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: '8'.repeat(32),+ name: 'missing-reconcile.bin',+ sizeBytes: bytes.length,+ idempotencyKey: randomUUID(),+ })+ await uploads.putPart(+ committed.session.uploadId,+ partGeometry({ expectedBytes: bytes.length, partBytes: CONFIG.preferredPartBytes }, 0),+ checksum(bytes),+ Readable.from([bytes]) as never,+ { drive, ownerSubject: owner, primarySubject: owner }+ )+ const partRow = await pool.query<{ staging_path: string }>(+ `SELECT staging_path FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed'`,+ [committed.session.uploadId]+ )+ await rm(String(partRow.rows[0]?.staging_path), { force: true })++ const corruptResult = await uploads.reconcile()+ expect(corruptResult).toMatchObject({ staleParts: 0 })+ const corruptRow = await pool.query<{ state: string; failure_code: string }>(+ `SELECT state, failure_code FROM gfs_upload_sessions WHERE upload_id = $1`,+ [committed.session.uploadId]+ )+ expect(corruptRow.rows[0]).toEqual({ state: 'failed', failure_code: 'corrupt_part_missing' })+ })+it('rejects cancel during finalization and leaves one completed receipt', async () => {let finalizerStarted!: () => voidlet releaseFinalizer!: () => voiddiff --git a/control-api/test/routes.gfsProxy.test.ts b/control-api/test/routes.gfsProxy.test.tsindex ae76ecfab..cb8053879 100644--- a/control-api/test/routes.gfsProxy.test.ts+++ b/control-api/test/routes.gfsProxy.test.ts@@ -199,4 +199,53 @@ describe('/api/v1/gfs/proxy', () => {expect(res.status).toBe(413)expect(res.body).toEqual({ ok: false, error: { code: 'payload_too_large' } })})++ it('preserves upload-length and Retry-After headers for lifecycle clients', async () => {+ const fetchMock = vi.fn(+ async () =>+ new Response('{"error":"quota_exceeded"}', {+ status: 429,+ headers: {+ 'content-type': 'application/json',+ 'retry-after': '3',+ 'upload-length': '209715200',+ },+ })+ )+ vi.stubGlobal('fetch', fetchMock)++ const app = await buildApp()+ const res = await request(app).post('/api/v1/gfs/proxy/v1/uploads').send({})++ expect(res.status).toBe(429)+ expect(res.headers['retry-after']).toBe('3')+ expect(res.headers['upload-length']).toBe('209715200')+ })++ it('preserves the bounded upload response headers on successful lifecycle responses', async () => {+ const fetchMock = vi.fn(+ async () =>+ new Response(null, {+ status: 204,+ headers: {+ location: '/v1/uploads/01234567-89ab-cdef-0123-456789abcdef',+ 'upload-offset': '8388608',+ 'upload-active-parts': '1',+ 'upload-state': 'uploading',+ },+ })+ )+ vi.stubGlobal('fetch', fetchMock)++ const app = await buildApp()+ const res = await request(app).head(+ '/api/v1/gfs/proxy/v1/uploads/01234567-89ab-cdef-0123-456789abcdef'+ )++ expect(res.status).toBe(204)+ expect(res.headers.location).toContain('/v1/uploads/')+ expect(res.headers['upload-offset']).toBe('8388608')+ expect(res.headers['upload-active-parts']).toBe('1')+ expect(res.headers['upload-state']).toBe('uploading')+ })})diff --git a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts b/control-api/test/services.rateLimiter.realPostgres.integration.test.tsnew file mode 100644index 000000000..2879879ce--- /dev/null+++ b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts@@ -0,0 +1,100 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomBytes } from 'node:crypto'+import { Pool } from 'pg'+import { initDb } from '../src/db.js'+import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++describeRealPostgres('rate limiter atomicity on real PostgreSQL', () => {+ const database = `rate_limiter_${randomBytes(6).toString('hex')}`+ const bucketKey = `real-pg:${randomBytes(8).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ const windowStartMs = Math.floor(Date.now() / 60_000) * 60_000+ let adminPool: Pool+ let pool: Pool++ beforeAll(async () => {+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ pool = new Pool({ connectionString })+ await initDb({ connect: () => pool.connect() })+ }, 60_000)++ afterAll(async () => {+ await pool?.query('DELETE FROM rate_limit_buckets WHERE bucket_key = $1', [bucketKey])+ await pool?.end()+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ it('returns distinct post-increment counts under concurrent requests', async () => {+ const results = await Promise.all(+ Array.from({ length: 32 }, () =>+ checkAndIncrementWithQuery(+ (text, values) => pool.query(text, values),+ bucketKey,+ 32,+ windowStartMs,+ 1+ )+ )+ )+ const counts = results.map(result => result.count).sort((a, b) => a - b)+ expect(counts).toEqual(Array.from({ length: 32 }, (_, index) => index + 1))+ const persisted = await pool.query<{ count: string }>(+ 'SELECT count FROM rate_limit_buckets WHERE bucket_key = $1 AND window_start_ms = $2',+ [bucketKey, windowStartMs]+ )+ expect(Number(persisted.rows[0]?.count)).toBe(32)+ })++ it('serializes a bounded advisory slot across independent PostgreSQL sessions', async () => {+ const key = `gfs-upload-slot:${randomBytes(8).toString('hex')}`+ const first = await pool.connect()+ const second = await pool.connect()+ try {+ const acquiredFirst = await first.query<{ acquired: boolean }>(+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,+ [key]+ )+ const acquiredSecond = await second.query<{ acquired: boolean }>(+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,+ [key]+ )+ expect(acquiredFirst.rows[0]?.acquired).toBe(true)+ expect(acquiredSecond.rows[0]?.acquired).toBe(false)++ await first.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])+ const acquiredAfterRelease = await second.query<{ acquired: boolean }>(+ `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,+ [key]+ )+ expect(acquiredAfterRelease.rows[0]?.acquired).toBe(true)+ await second.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])+ } finally {+ first.release()+ second.release()+ }+ })+})diff --git a/control-ui/components/GfsBrowser.tsx b/control-ui/components/GfsBrowser.tsxindex 29210735b..86df75d0d 100644--- a/control-ui/components/GfsBrowser.tsx+++ b/control-ui/components/GfsBrowser.tsx@@ -173,6 +173,44 @@ export function GfsBrowser(): React.JSX.Element {// Resource IDs currently streaming a download (disables that row's button).const [downloadingIds, setDownloadingIds] = useState<ReadonlySet<string>>(() => new Set())+ /**+ * The uploader already emits the exact aggregate of committed bytes plus+ * each part's maximum in-flight observation. Keep a small monotonic guard+ * only for two stale `uploading` snapshots arriving out of order; terminal+ * or paused snapshots, and direct progress observations, must be accepted+ * exactly so a failed retry cannot leave the bar overstated.+ */+ const mergeUploadSnapshot = useCallback((next: GfsUploadJobSnapshot): void => {+ setUploadSnapshot(previous => {+ if (!previous) return next+ const previousUploadId = previous.session?.uploadId+ const nextUploadId = next.session?.uploadId+ if (+ (previousUploadId && nextUploadId && previousUploadId !== nextUploadId) ||+ previous.totalBytes !== next.totalBytes+ )+ return next+ const uploadedBytes =+ previous.state === 'uploading' && next.state === 'uploading'+ ? Math.max(previous.uploadedBytes, next.uploadedBytes)+ : next.uploadedBytes+ return {+ ...next,+ uploadedBytes: Math.min(Math.max(0, uploadedBytes), next.totalBytes),+ }+ })+ }, [])++ const mergeUploadProgress = useCallback((uploadedBytes: number): void => {+ setUploadSnapshot(previous => {+ if (!previous) return previous+ return {+ ...previous,+ uploadedBytes: Math.min(Math.max(0, uploadedBytes), previous.totalBytes),+ }+ })+ }, [])+const current = crumbs[crumbs.length - 1]const currentLabel = current?.name === '/' ? DRIVE : current?.name || DRIVE@@ -284,13 +322,11 @@ export function GfsBrowser(): React.JSX.Element {const job = createGfsUploadJob({...input,onProgress: progress => {- setUploadSnapshot(previous =>- previous ? { ...previous, uploadedBytes: progress.uploadedBytes } : previous- )+ mergeUploadProgress(progress.uploadedBytes)input.onProgress?.(progress)},onState: snapshot => {- setUploadSnapshot(snapshot)+ mergeUploadSnapshot(snapshot)input.onState?.(snapshot)},onPersistPending: record => {@@ -412,11 +448,9 @@ export function GfsBrowser(): React.JSX.Element {})},onProgress: progress => {- setUploadSnapshot(previous =>- previous ? { ...previous, uploadedBytes: progress.uploadedBytes } : previous- )+ mergeUploadProgress(progress.uploadedBytes)},- onState: snapshot => setUploadSnapshot(snapshot),+ onState: snapshot => mergeUploadSnapshot(snapshot),onPersist: record => {persistPendingGfsUpload({uploadId: record.uploadId,diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsxindex dabf15c80..57d3f5e14 100644--- a/control-ui/components/__tests__/GfsBrowser.test.tsx+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx@@ -367,9 +367,23 @@ describe('GfsBrowser', () => {}const paused = { ...completed, state: 'paused', committedBytes: 2 }let resolveStart: ((value: typeof completed) => void) | undefined- let uploadInput: { onState?: (snapshot: unknown) => void } | undefined+ let uploadInput:+ | {+ onState?: (snapshot: unknown) => void+ onProgress?: (progress: { uploadedBytes: number }) => void+ }+ | undefinedconst job = {start: vi.fn(() => {+ uploadInput?.onState?.({+ state: 'uploading',+ session: { ...completed, state: 'uploading' },+ uploadedBytes: 2,+ totalBytes: 4,+ })+ // Simulate an out-of-order in-flight part reporting a later absolute+ // offset before the accumulator commits the lower-numbered part.+ uploadInput?.onProgress?.({ uploadedBytes: 3 })uploadInput?.onState?.({state: 'uploading',session: { ...completed, state: 'uploading' },@@ -408,7 +422,10 @@ describe('GfsBrowser', () => {cancel: vi.fn(async () => undefined),}mockCreateGfsUploadJob.mockImplementationOnce(- (input: { onState?: (snapshot: unknown) => void }) => {+ (input: {+ onState?: (snapshot: unknown) => void+ onProgress?: (progress: { uploadedBytes: number }) => void+ }) => {uploadInput = inputreturn job}@@ -424,10 +441,10 @@ describe('GfsBrowser', () => {const progress = await within(uploadDialog).findByRole('progressbar', {name: /Upload progress/,})- expect(progress).toHaveAttribute('value', '2')+ expect(progress).toHaveAttribute('value', '3')expect(progress).toHaveAttribute('aria-valuemin', '0')expect(progress).toHaveAttribute('aria-valuemax', '4')- expect(progress).toHaveAttribute('aria-valuenow', '2')+ expect(progress).toHaveAttribute('aria-valuenow', '3')fireEvent.click(within(uploadDialog).getByRole('button', { name: 'Pause' }))await waitFor(() => expect(job.pause).toHaveBeenCalledTimes(1))expect(await within(uploadDialog).findByRole('button', { name: 'Resume' })).toBeTruthy()diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.tsindex d21787b80..eac70e826 100644--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts@@ -1,6 +1,11 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import { GFS_FILE_UPLOAD_MAX_BYTES, GFS_FILE_UPLOAD_MAX_MEGABYTES } from '@constants/gfsFileUpload'-import { GfsUploadJob, type GfsUploadStatus, assertGfsFileUploadSize } from '@lib/gfsFileUpload'+import {+ GfsUploadJob,+ type GfsUploadStatus,+ assertGfsFileUploadSize,+ parseRetryAfter,+} from '@lib/gfsFileUpload'describe('assertGfsFileUploadSize', () => {it('accepts files at the upload limit', () => {@@ -14,6 +19,14 @@ describe('assertGfsFileUploadSize', () => {})})+describe('parseRetryAfter', () => {+ it('accepts delta-seconds and caps untrusted server delays', () => {+ expect(parseRetryAfter('2')).toBe(2_000)+ expect(parseRetryAfter('60')).toBe(5_000)+ expect(parseRetryAfter('not-a-date')).toBeUndefined()+ })+})+class FakeXhr {static requests: FakeXhr[] = []static statuses: number[] = []@@ -32,9 +45,13 @@ class FakeXhr {setRequestHeader(name: string, value: string): void {this.headers.set(name, value)}+ getResponseHeader(name: string): string | null {+ return this.headers.get(name) ?? null+ }send(body: Blob): void {this.body = bodythis.status = FakeXhr.statuses.shift() ?? 204+ if (this.status === 429) this.headers.set('retry-after', '0')FakeXhr.requests.push(this)void body.arrayBuffer().then(() => {this.upload.onprogress?.({@@ -68,6 +85,78 @@ describe('GfsUploadJob', () => {vi.restoreAllMocks()})+ it('honors the writer-advertised maxFileBytes before creating a session', async () => {+ const file = new File([new Uint8Array([1, 2])], 'ceiling.bin')+ let createCalls = 0+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ if (method === 'GET' && String(_input).endsWith('/capabilities'))+ return new Response(+ JSON.stringify({ upload: { resumableV2: { enabled: true, maxFileBytes: 1 } } }),+ { status: 200 }+ )+ if (method === 'POST' && String(_input).endsWith('/uploads')) createCalls += 1+ return new Response(null, { status: 204 })+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-ceiling' },+ }).start()+ ).rejects.toThrow('writer limit is 1 bytes')+ expect(createCalls).toBe(0)+ })++ it('rejects a create receipt that changes the canonical control-ui drive', async () => {+ const file = new File([new Uint8Array([3])], 'drive-drift.bin')+ let partCalls = 0+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/uploads'))+ return new Response(+ JSON.stringify({+ ok: true,+ data: {+ uploadId: '18181818-1818-4181-8181-181818181818',+ drive: 'archive',+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ },+ }),+ { status: 201 }+ )+ if (method === 'PUT') partCalls += 1+ return new Response(null, { status: 204 })+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-drive-drift' },+ }).start()+ ).rejects.toThrow('upload_drive_mismatch')+ expect(partCalls).toBe(0)+ })+it('resumes only the missing part and emits visible progress before completion', async () => {const file = new File([new Uint8Array([1, 2, 3, 4])], 'payload.bin', { lastModified: 12 })const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', new Uint8Array([1, 2])))@@ -139,6 +228,106 @@ describe('GfsUploadJob', () => {expect(responses).toHaveLength(0)})+ it('retries a lifecycle 429 using the server Retry-After header', async () => {+ const file = new File([new Uint8Array([21])], 'retry-after.bin')+ const session = {+ uploadId: '12121212-1212-4121-8121-121212121212',+ drive: 'main',+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ }+ let createCalls = 0+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/uploads')) {+ createCalls += 1+ if (createCalls === 1)+ return new Response('{"error":"quota_exceeded"}', {+ status: 429,+ headers: { 'retry-after': '0' },+ })+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })+ }+ if (method === 'POST' && url.endsWith('/complete'))+ return new Response(+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),+ {+ status: 200,+ }+ )+ throw new Error(`unexpected ${method} ${url}`)+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-retry-after' },+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(createCalls).toBe(2)+ })++ it('retries an XHR part 429 using the server Retry-After header', async () => {+ const file = new File([new Uint8Array([22])], 'part-retry-after.bin')+ const session = {+ uploadId: '13131313-1313-4131-8131-131313131314',+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ }+ FakeXhr.statuses = [429, 204]+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/uploads'))+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })+ if (method === 'POST' && url.endsWith('/complete'))+ return new Response(+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),+ {+ status: 200,+ }+ )+ throw new Error(`unexpected ${method} ${url}`)+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-part-retry-after' },+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(FakeXhr.requests).toHaveLength(2)+ })+it('adopts a status-confirmed committed part after its XHR response is lost', async () => {const file = new File([new Uint8Array([5, 6, 7, 8])], 'response-loss.bin', {lastModified: 13,@@ -416,8 +605,8 @@ describe('GfsUploadJob', () => {upload: {resumableV2: {enabled: true,- preferredPartBytes: 2,- maxPartBytes: 2,+ preferredChunkBytes: 2,+ maxChunkBytes: 2,maxConcurrentPartsPerSession: 4,fallbackConcurrency: 2,},@@ -508,7 +697,7 @@ describe('GfsUploadJob', () => {if (init?.method === 'GET' && String(_input).endsWith('/capabilities'))return new Response(JSON.stringify({- upload: { resumableV2: { enabled: true, preferredPartBytes: 2, maxPartBytes: 2 } },+ upload: { resumableV2: { enabled: true, preferredChunkBytes: 2, maxChunkBytes: 2 } },}),{ status: 200 })diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.tsindex dec63d25f..7734ec689 100644--- a/control-ui/lib/gfsFileUpload.ts+++ b/control-ui/lib/gfsFileUpload.ts@@ -8,7 +8,11 @@ import {import { apiSend } from './api'const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'+const GFS_UPLOAD_DRIVE = 'main'const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])+export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3+export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000+const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60_000const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3@@ -108,8 +112,7 @@ interface UploadCapabilities {upload?: {resumableV2?: {enabled?: boolean- preferredPartBytes?: number- maxPartBytes?: number+ maxFileBytes?: numberpreferredChunkBytes?: numbermaxChunkBytes?: numbermaxConcurrentPartsPerSession?: number@@ -193,7 +196,7 @@ async function requestJson<T>(async function uploadError(response: Response-): Promise<Error & { status?: number; code?: string }> {+): Promise<Error & { status?: number; code?: string; retryAfterMs?: number }> {const text = await response.text().catch(() => '')let message = text || response.statusText || 'GFS upload failed'let code: string | undefined@@ -218,19 +221,44 @@ async function uploadError(const error = new Error(`${response.status} ${message}`) as Error & {status?: numbercode?: string+ retryAfterMs?: number}error.status = response.statuserror.code = code+ error.retryAfterMs = parseRetryAfter(response.headers.get('retry-after'))return error}+/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */+export function parseRetryAfter(+ value: string | null | undefined,+ nowMs = Date.now(),+ capMs = GFS_UPLOAD_RETRY_AFTER_CAP_MS+): number | undefined {+ if (value === null || value === undefined) return undefined+ const trimmed = value.trim()+ if (!trimmed) return undefined+ const seconds = Number(trimmed)+ let delayMs: number+ if (Number.isFinite(seconds) && seconds >= 0) {+ delayMs = seconds * 1_000+ } else {+ const dateMs = Date.parse(trimmed)+ if (!Number.isFinite(dateMs)) return undefined+ delayMs = Math.max(0, dateMs - nowMs)+ }+ if (!Number.isFinite(delayMs) || delayMs < 0) return undefined+ return Math.min(capMs, delayMs)+}+function assertRetryable(error: unknown): boolean {if (error instanceof Error &&(error.name === 'AbortError' ||error.message.startsWith('source_changed:') ||error.message.startsWith('upload_part_mismatch:') ||- error.message.startsWith('upload_part_outcome_unknown:'))+ error.message.startsWith('upload_part_outcome_unknown:') ||+ error.message.startsWith('upload_completion_outcome_unknown:')))return falseconst status =@@ -256,6 +284,92 @@ function delay(ms: number, signal?: AbortSignal): Promise<void> {})}+function retryAfterOf(error: unknown): number | undefined {+ if (!error || typeof error !== 'object') return undefined+ const value = error as {+ retryAfterMs?: unknown+ retryAfter?: unknown+ headers?: { get?: (name: string) => string | null }+ response?: { headers?: { get?: (name: string) => string | null } }+ }+ if (typeof value.retryAfterMs === 'number' && Number.isFinite(value.retryAfterMs))+ return Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, Math.max(0, value.retryAfterMs))+ const header =+ typeof value.retryAfter === 'string'+ ? value.retryAfter+ : (value.headers?.get?.('retry-after') ?? value.response?.headers?.get?.('retry-after'))+ return parseRetryAfter(typeof header === 'string' ? header : undefined)+}++function ambiguousLifecycleError(error: unknown): boolean {+ const status = statusOf(error)+ return status === undefined || status === 408 || (status !== undefined && status >= 500)+}++function terminalReconciliationError(error: unknown): boolean {+ if (!(error instanceof Error)) return false+ return [+ 'upload status changed during reconciliation',+ 'upload_part_mismatch:',+ 'source_changed:',+ 'upload_completion_outcome_unknown:',+ 'cannot reconcile ',+ ].some(prefix => error.message.startsWith(prefix))+}++interface LifecycleRecovery<T> {+ value: T+}++interface LifecycleRetryOptions<T> {+ signal?: AbortSignal+ attempts?: number+ reconcileOnError?: (error: unknown) => boolean+ reconcile?: (error: unknown) => Promise<LifecycleRecovery<T> | undefined>+}++/**+ * Bounded lifecycle retry primitive. It centralizes status classification,+ * capped Retry-After handling, abort propagation, and optional ambiguity+ * reconciliation for state-changing requests.+ */+async function requestWithLifecycleRetries<T>(+ operation: () => Promise<T>,+ options: LifecycleRetryOptions<T> = {}+): Promise<T> {+ const attempts = Math.max(1, options.attempts ?? GFS_UPLOAD_RETRY_MAX_ATTEMPTS)+ let lastError: unknown+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ try {+ return await operation()+ } catch (error) {+ lastError = error+ if (options.signal?.aborted) throw options.signal.reason ?? error+ const shouldReconcile = options.reconcile+ ? (options.reconcileOnError?.(error) ?? ambiguousLifecycleError(error))+ : false+ if (shouldReconcile) {+ try {+ const recovered = await options.reconcile!(error)+ if (recovered) return recovered.value+ } catch (reconcileError) {+ // A transient status/HEAD failure must not suppress the bounded+ // retry of the original request. Definitive geometry, auth, or+ // outcome-unknown errors remain terminal and are surfaced.+ if (terminalReconciliationError(reconcileError)) throw reconcileError+ }+ }+ if (!assertRetryable(error) || attempt + 1 >= attempts) throw error+ await delay(+ retryAfterOf(error) ??+ Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ options.signal+ )+ }+ }+ throw lastError+}+function digestBase64(bytes: ArrayBuffer): string {let binary = ''const view = new Uint8Array(bytes)@@ -309,16 +423,21 @@ function indexCommittedParts(async function loadUploadStatus(uploadId: string,- expected: { expectedBytes: number; partBytes?: number; partCount?: number },+ expected: { expectedBytes: number; drive?: string; partBytes?: number; partCount?: number },signal?: AbortSignal,- timeoutMs = 600_000+ timeoutMs = 600_000,+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS): Promise<{ status: GfsUploadStatus; head: Response }> {- const head = await requestJson<undefined>(- 'HEAD',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}`,- undefined,- signal,- timeoutMs+ const head = await requestWithLifecycleRetries(+ () =>+ requestJson<undefined>(+ 'HEAD',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}`,+ undefined,+ signal,+ timeoutMs+ ),+ { signal, attempts: retryAttempts })const declaredLengthHeader = head.response.headers.get('upload-length')if (declaredLengthHeader !== null) {@@ -332,12 +451,16 @@ async function loadUploadStatus(const parts: GfsUploadPartReceipt[] = []for (let page = 0; page < 8; page += 1) {const query = cursor ? `?limit=256&cursor=${encodeURIComponent(cursor)}` : '?limit=256'- const response = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(- 'GET',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}/status${query}`,- undefined,- signal,- timeoutMs+ const response = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: GfsUploadStatus }>(+ 'GET',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(uploadId)}/status${query}`,+ undefined,+ signal,+ timeoutMs+ ),+ { signal, attempts: retryAttempts })const current = receiptFromResponse(response)session ??= current.session@@ -346,6 +469,9 @@ async function loadUploadStatus(session.uploadId !== uploadId ||session.expectedBytes !== current.session.expectedBytes ||session.expectedBytes !== expected.expectedBytes ||+ (expected.drive !== undefined &&+ current.session.drive !== undefined &&+ current.session.drive !== expected.drive) ||session.partBytes !== current.session.partBytes ||session.partCount !== current.session.partCount ||(expected.partBytes !== undefined && current.session.partBytes !== expected.partBytes) ||@@ -388,11 +514,13 @@ async function reconcileAmbiguousPart(session.uploadId,{expectedBytes: file.size,+ drive: GFS_UPLOAD_DRIVE,partBytes: session.partBytes,partCount: session.partCount,},signal,- GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,+ 1)).status} catch (error) {@@ -475,8 +603,9 @@ async function putPart(}const error = new Error(`${xhr.status} ${xhr.responseText || 'GFS upload failed'}`- ) as Error & { status?: number }+ ) as Error & { status?: number; retryAfterMs?: number }error.status = xhr.status+ error.retryAfterMs = parseRetryAfter(xhr.getResponseHeader('retry-after'))reject(error)}xhr.onerror = () => reject(new Error('GFS upload network error'))@@ -534,7 +663,10 @@ async function uploadPartWithRetries(}}if (attempt === 2) throw error- await delay(Math.min(1000, 150 * (attempt + 1)), signal)+ await delay(+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ signal+ )}}throw lastError@@ -549,6 +681,7 @@ async function runParts(shouldPause: () => boolean,onProgress?: (progress: GfsUploadProgress) => void,onByteProgress?: (progress: GfsUploadProgress) => void,+ onPartFailure?: (partNumber: number) => void,fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY): Promise<{ failed: typeof parts; retryableFailures: number; paused: boolean }> {const initialConcurrency = Math.max(1, Math.min(concurrency, parts.length || 1))@@ -607,6 +740,7 @@ async function runParts(failed.delete(part.partNumber)retryableFailed.delete(part.partNumber)} catch (error) {+ onPartFailure?.(part.partNumber)if (shouldPause()) {return}@@ -650,6 +784,11 @@ function statusOf(error: unknown): number | undefined {return value && typeof value.status === 'number' ? value.status : undefined}+function errorCodeOf(error: unknown): string | undefined {+ const value = error as { code?: unknown } | null+ return value && typeof value.code === 'string' ? value.code : undefined+}+export class GfsUploadJob {private readonly abortController = new AbortController()private readonly idempotencyKey = uuid()@@ -700,13 +839,19 @@ export class GfsUploadJob {throw new Error(`cannot pause upload in ${session.state}`)}this.pauseRequested = true- const response = await requestJson<{ ok: boolean; data: UploadReceipt }>(- 'POST',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/pause`,- {},- undefined+ const paused = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: UploadReceipt }>(+ 'POST',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/pause`,+ {},+ this.abortController.signal+ ).then(receiptFromResponse),+ {+ signal: this.abortController.signal,+ reconcile: () => this.reconcileMutation(session, 'pause'),+ })- const paused = receiptFromResponse(response)this.session = pausedthis.state = 'paused'this.emit()@@ -721,13 +866,20 @@ export class GfsUploadJob {throw new Error(`cannot resume upload in ${session.state}`)}this.pauseRequested = false- const response = await requestJson<{ ok: boolean; data: UploadReceipt }>(- 'POST',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/resume`,- {},- this.abortController.signal+ const resumed = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: UploadReceipt }>(+ 'POST',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/resume`,+ {},+ this.abortController.signal+ ).then(receiptFromResponse),+ {+ signal: this.abortController.signal,+ reconcile: () => this.reconcileMutation(session, 'resume'),+ })- this.session = receiptFromResponse(response)+ this.session = resumedthis.state = 'uploading'this.emit()return this.start()@@ -741,40 +893,52 @@ export class GfsUploadJob {this.state = 'canceling'this.emit()if (session && !['completed', 'aborted'].includes(session.state)) {+ let reconciledCompleted: UploadReceipt | null = nulltry {- await requestJson(- 'DELETE',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}`,- undefined,- undefined+ await requestWithLifecycleRetries(+ () =>+ requestJson(+ 'DELETE',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}`,+ undefined,+ undefined+ ).then(() => undefined),+ {+ reconcileOnError: error =>+ ambiguousLifecycleError(error) ||+ statusOf(error) === 409 ||+ errorCodeOf(error) === 'upload_finalizing',+ reconcile: async error => {+ const observed = await this.observeSession(+ session,+ GFS_UPLOAD_RETRY_MAX_ATTEMPTS,+ undefined+ )+ this.session = observed+ if (observed.state === 'completed') {+ reconciledCompleted = observed+ return { value: undefined }+ }+ if (observed.state === 'aborted') return { value: undefined }+ if (observed.state === 'finalizing') throw error+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined+ throw new Error(`cannot reconcile cancel upload in ${observed.state}`)+ },+ })} catch (error) {- if (- statusOf(error) === 409 ||- (typeof error === 'object' &&- error !== null &&- (error as { code?: unknown }).code === 'upload_finalizing')- ) {- const reconciled = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(- 'GET',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(session.uploadId)}/status`,- undefined,- undefined- )- this.session = receiptFromResponse(reconciled).session- if (this.session.state === 'completed') {- this.state = 'completed'- this.canceled = false- this.emit()- await this.input.onClearPersisted?.(session.uploadId)- return- }- }this.canceled = falsethis.state = 'failed'this.emit()throw error}+ if (reconciledCompleted) {+ this.state = 'completed'+ this.canceled = false+ this.emit()+ await this.input.onClearPersisted?.(session.uploadId)+ return+ }}await this.runPromise?.catch(() => undefined)this.state = 'aborted'@@ -800,14 +964,76 @@ export class GfsUploadJob {}}+ private async observeSession(+ session: UploadReceipt,+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS,+ signal: AbortSignal | undefined = this.abortController.signal+ ): Promise<UploadReceipt> {+ return (+ await loadUploadStatus(+ session.uploadId,+ {+ expectedBytes: this.input.file.size,+ drive: GFS_UPLOAD_DRIVE,+ partBytes: session.partBytes,+ partCount: session.partCount,+ },+ signal,+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,+ retryAttempts+ )+ ).status.session+ }++ private async reconcileMutation(+ session: UploadReceipt,+ action: 'pause' | 'resume'+ ): Promise<LifecycleRecovery<UploadReceipt> | undefined> {+ const observed = await this.observeSession(session)+ if (+ (action === 'pause' && observed.state === 'paused') ||+ (action === 'resume' && ['initiated', 'uploading', 'completed'].includes(observed.state))+ ) {+ return { value: observed }+ }+ if (+ (action === 'pause' && ['initiated', 'uploading'].includes(observed.state)) ||+ (action === 'resume' && observed.state === 'paused')+ ) {+ return undefined+ }+ throw new Error(`cannot reconcile ${action} upload in ${observed.state}`)+ }++ private async reconcileCompletion(+ session: UploadReceipt+ ): Promise<LifecycleRecovery<UploadReceipt> | undefined> {+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {+ const observed = await this.observeSession(session, 1)+ if (observed.state === 'completed') return { value: observed }+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined+ if (observed.state !== 'finalizing')+ throw new Error(`cannot reconcile upload completion in ${observed.state}`)+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)+ await delay(GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt, this.abortController.signal)+ }+ throw new Error(+ 'upload_completion_outcome_unknown: finalization is still in progress; resume the persisted session'+ )+ }+private async loadCapabilities(): Promise<UploadCapabilities['upload']> {let capabilities: { data: UploadCapabilities }try {- capabilities = await requestJson<UploadCapabilities>(- 'GET',- '/api/v1/gfs/proxy/v1/capabilities',- undefined,- this.abortController.signal+ capabilities = await requestWithLifecycleRetries(+ () =>+ requestJson<UploadCapabilities>(+ 'GET',+ '/api/v1/gfs/proxy/v1/capabilities',+ undefined,+ this.abortController.signal+ ),+ { signal: this.abortController.signal })} catch (error) {throw new GfsUploadCapabilityError('Resumable GFS upload capabilities are unavailable.', {@@ -827,12 +1053,20 @@ export class GfsUploadJob {}> {const upload = await this.loadCapabilities()const resumable = upload.resumableV2!+ if (resumable.maxFileBytes !== undefined) {+ if (!Number.isSafeInteger(resumable.maxFileBytes) || resumable.maxFileBytes < 0)+ throw new GfsUploadCapabilityError(+ 'Resumable GFS upload capabilities advertised an invalid file ceiling.'+ )+ if (this.input.file.size > resumable.maxFileBytes)+ throw new Error(`GFS writer limit is ${resumable.maxFileBytes} bytes for this upload.`)+ }const resumeId = this.session?.uploadId ?? this.input.resumeUploadIdif (resumeId) {const { session, parts } = (await loadUploadStatus(resumeId,- { expectedBytes: this.input.file.size },+ { expectedBytes: this.input.file.size, drive: GFS_UPLOAD_DRIVE },this.abortController.signal)).status@@ -858,27 +1092,21 @@ export class GfsUploadJob {target: this.input.target,name: this.input.name,})- let create: { data: { ok: boolean; data: UploadReceipt }; response: Response }- try {- create = await requestJson<{ ok: boolean; data: UploadReceipt }>(- 'POST',- '/api/v1/gfs/proxy/v1/uploads',- createBody,- this.abortController.signal- )- } catch (error) {- // A response lost after POST is safe to replay because the key is fixed.- if (statusOf(error) !== undefined) throw error- create = await requestJson<{ ok: boolean; data: UploadReceipt }>(- 'POST',- '/api/v1/gfs/proxy/v1/uploads',- createBody,- this.abortController.signal- )- }+ const create = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: UploadReceipt }>(+ 'POST',+ '/api/v1/gfs/proxy/v1/uploads',+ createBody,+ this.abortController.signal+ ),+ { signal: this.abortController.signal }+ )const session = receiptFromResponse(create)if (session.expectedBytes !== this.input.file.size)throw new Error('GFS upload session size changed before transfer')+ if (session.drive !== undefined && session.drive !== GFS_UPLOAD_DRIVE)+ throw new Error('upload_drive_mismatch: writer changed drive')return { session, committed: new Set<number>(), resumable }}@@ -938,12 +1166,49 @@ export class GfsUploadJob {}))const allParts = partsconst completedParts = new Set(prepared.committed)- const progress = (value: GfsUploadProgress) => {- completedParts.add(value.partNumber)- this.uploadedBytes = allParts.reduce(+ const inFlightLoaded = new Map<number, number>()+ const calculateUploadedBytes = (): number => {+ const committedBytes = allParts.reduce((total, part) => total + (completedParts.has(part.partNumber) ? part.lengthBytes : 0),0)+ const inFlightBytes = [...inFlightLoaded.entries()].reduce(+ (total, [partNumber, loaded]) => total + (completedParts.has(partNumber) ? 0 : loaded),+ 0+ )+ const observed = Math.min(this.input.file.size, committedBytes + inFlightBytes)+ // Part bytes are committed before the final resource publication. Do+ // not expose a visually complete bar until the complete receipt is+ // durable; the caller sets the exact total only after that response.+ return Math.min(Math.max(0, this.input.file.size - 1), observed)+ }+ const progress = (value: GfsUploadProgress) => {+ completedParts.add(value.partNumber)+ inFlightLoaded.delete(value.partNumber)+ this.uploadedBytes = calculateUploadedBytes()+ this.input.onProgress?.({+ ...value,+ uploadedBytes: this.uploadedBytes,+ partCount: allParts.length,+ })+ this.emit()+ }+ const byteProgress = (value: GfsUploadProgress) => {+ const part = allParts[value.partNumber]+ if (!part) return+ const loaded = Math.min(+ part.lengthBytes,+ Math.max(0, value.uploadedBytes - part.offsetBytes)+ )+ inFlightLoaded.set(+ value.partNumber,+ Math.max(inFlightLoaded.get(value.partNumber) ?? 0, loaded)+ )+ // A progress event is an observation of bytes sent, not a commit+ // receipt. Deduplicate each part's in-flight contribution, and remove+ // it when the worker reports a failed attempt so a retry cannot leave+ // the aggregate permanently overstated.+ this.uploadedBytes = calculateUploadedBytes()this.input.onProgress?.({...value,uploadedBytes: this.uploadedBytes,@@ -967,30 +1232,26 @@ export class GfsUploadJob {this.abortController.signal,() => this.pauseRequested || this.canceled,progress,- value => this.input.onProgress?.(value),+ byteProgress,+ partNumber => {+ inFlightLoaded.delete(partNumber)+ this.uploadedBytes = calculateUploadedBytes()+ },fallbackConcurrency)if (result.paused || this.pauseRequested) {this.state = 'paused'- const paused = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(- 'GET',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,- undefined,- undefined- )- this.session = receiptFromResponse(paused).session- this.emit()- return this.session- }- if (result.paused || this.pauseRequested) {- this.state = 'paused'- const paused = await requestJson<{ ok: boolean; data: GfsUploadStatus }>(- 'GET',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,- undefined,- undefined+ const paused = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: GfsUploadStatus }>(+ 'GET',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,+ undefined,+ this.abortController.signal+ ).then(receiptFromResponse),+ { signal: this.abortController.signal })- this.session = receiptFromResponse(paused).session+ this.session = paused.sessionthis.emit()return this.session}@@ -998,13 +1259,22 @@ export class GfsUploadJob {throw new Error('One or more upload parts could not be committed.')this.state = 'finalizing'this.emit()- const completedResponse = await requestJson<{ ok: boolean; data: UploadReceipt }>(- 'POST',- `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`,- {},- this.abortController.signal+ const completed = await requestWithLifecycleRetries(+ () =>+ requestJson<{ ok: boolean; data: UploadReceipt }>(+ 'POST',+ `/api/v1/gfs/proxy/v1/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`,+ {},+ this.abortController.signal+ ).then(receiptFromResponse),+ {+ signal: this.abortController.signal,+ reconcileOnError: error =>+ ambiguousLifecycleError(error) || errorCodeOf(error) === 'upload_finalizing',+ reconcile: () => this.reconcileCompletion(prepared.session),+ })- this.session = receiptFromResponse(completedResponse)+ this.session = completedthis.state = 'completed'this.uploadedBytes = this.input.file.sizethis.input.onProgress?.({diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.tsindex 2d6565e2c..fa12dbd6b 100644--- a/desktop-app/src/__tests__/appService.gfs.test.ts+++ b/desktop-app/src/__tests__/appService.gfs.test.ts@@ -247,6 +247,12 @@ function authenticatedUploadService(statePath: string): UploadScopeTestService {}service.gfsAuthEpoch = 7service.gfsDispatchBlocked = false+ ;(service as any).gfsScopeIdentity = {+ ownerId: service.me.id,+ teamId: service.me.teamId,+ environmentKey: getActiveEnvKey(),+ baseUrl: canonicalExternalBaseUrl(),+ }service.desktopGfsUploadStatePath = async () => statePathservice.authClient = {passwordLogin: vi.fn(),@@ -430,6 +436,12 @@ describe('AppService GFS upload security scope', () => {role: null,}service.gfsDispatchBlocked = false+ ;(service as any).gfsScopeIdentity = {+ ownerId: service.me.id,+ teamId: service.me.teamId,+ environmentKey: getActiveEnvKey(),+ baseUrl: canonicalExternalBaseUrl(),+ }await expect(service.listGfsUploadSessions('main')).resolves.toEqual([])await expect(service.resumeGfsUpload(exact.uploadId, 'main')).rejects.toThrow('not available in the active security scope'@@ -447,6 +459,12 @@ describe('AppService GFS upload security scope', () => {role: null,}service.gfsAuthEpoch += 1+ ;(service as any).gfsScopeIdentity = {+ ownerId: service.me.id,+ teamId: service.me.teamId,+ environmentKey: getActiveEnvKey(),+ baseUrl: canonicalExternalBaseUrl(),+ }service.startDesktopGfsUpload.mockResolvedValue({ ...exact.session, state: 'uploading' })await service.resumeGfsUpload(exact.uploadId, 'main')expect(service.startDesktopGfsUpload).toHaveBeenCalledWith(diff --git a/desktop-app/src/__tests__/appService.rpc.test.ts b/desktop-app/src/__tests__/appService.rpc.test.tsindex 83c6f9988..78272c6a6 100644--- a/desktop-app/src/__tests__/appService.rpc.test.ts+++ b/desktop-app/src/__tests__/appService.rpc.test.ts@@ -147,6 +147,182 @@ describe('AppService.invokeHostMessage', () => {)})+ it('does not fence GFS uploads during a transient cross-team operation hop', async () => {+ const service = new AppService() as any+ service.desktopGfsUploadStatePath = vi+ .fn()+ .mockResolvedValue(path.join(chatStoreBaseDir, 'gfs-upload-state.json'))+ service.sessionToken = 'teamless-token'+ service.me = {+ id: 'user-1',+ email: '[email protected]',+ name: 'Test User',+ picture: null,+ teamId: 'team-a',+ teamName: 'Team A',+ role: 'member',+ }+ const uploadJob = { suspendForAuth: vi.fn() }+ service.gfsUploadJobs = new Map([+ [+ 'upload-1',+ {+ job: uploadJob,+ promise: Promise.resolve(undefined),+ scope: {+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ drive: 'main',+ authEpoch: 1,+ },+ },+ ],+ ])+ service.gfsPendingUploadJobs = new Set()+ service.gfsPendingLegacyUploads = new Set()+ service.authClient = {+ getMe: vi+ .fn()+ .mockResolvedValueOnce({ ...service.me, teamId: 'team-b', teamName: 'Team B' })+ .mockResolvedValueOnce({ ...service.me }),+ switchTeam: vi.fn(async (_token: string, teamId: string) => ({+ token: `${teamId}-token`,+ team: { id: teamId, name: teamId === 'team-b' ? 'Team B' : 'Team A', role: 'member' },+ })),+ }+ service.tokenStore = { setSessionToken: vi.fn() }++ await expect(+ service.runWithTeamContext('team-b', async (token: string) => token)+ ).resolves.toBe('team-b-token')++ expect(uploadJob.suspendForAuth).not.toHaveBeenCalled()+ expect(service.gfsUploadJobs).toHaveProperty('size', 1)+ expect(service.me.teamId).toBe('team-a')+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(1, 'teamless-token', 'team-b')+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(2, 'team-b-token', 'team-a')+ })++ it('fences the old GFS scope if a replacement team token cannot be refreshed', async () => {+ const service = new AppService() as any+ service.sessionToken = 'team-a-token'+ service.me = {+ id: 'user-1',+ email: '[email protected]',+ name: 'Test User',+ picture: null,+ teamId: 'team-a',+ teamName: 'Team A',+ role: 'member',+ }+ service.gfsScopeIdentity = {+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ }+ service.authClient = {+ switchTeam: vi.fn().mockResolvedValue({+ token: 'team-b-token',+ team: { id: 'team-b', name: 'Team B', role: 'member' },+ }),+ getMe: vi.fn().mockRejectedValue(new Error('replacement session rejected')),+ }+ service.tokenStore = { setSessionToken: vi.fn(), clearSessionToken: vi.fn() }+ service.rpcTokenManager = { clear: vi.fn() }+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn().mockResolvedValue(undefined)+ service.clearAuthenticatedSessionState = vi.fn()++ await expect(service.switchSessionToTeam('team-b', 'team-a-token')).rejects.toThrow(+ 'replacement session rejected'+ )++ expect(service.suspendDesktopGfsUploadsForAuthBoundary).toHaveBeenCalledWith({+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ })+ expect(service.clearAuthenticatedSessionState).toHaveBeenCalledTimes(1)+ })++ it('fences GFS uploads only after an explicit switchTeam succeeds', async () => {+ const service = new AppService() as any+ service.sessionToken = 'team-a-token'+ service.me = {+ id: 'user-1',+ email: '[email protected]',+ name: 'Test User',+ picture: null,+ teamId: 'team-a',+ teamName: 'Team A',+ role: 'member',+ }+ service.gfsScopeIdentity = {+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ }+ service.authClient = {+ switchTeam: vi.fn().mockResolvedValue({+ token: 'team-b-token',+ team: { id: 'team-b', name: 'Team B', role: 'member' },+ }),+ getMe: vi.fn().mockResolvedValue({ ...service.me, teamId: 'team-b', teamName: 'Team B' }),+ }+ service.tokenStore = { setSessionToken: vi.fn() }+ service.rpcTokenManager = { clear: vi.fn() }+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn().mockResolvedValue(undefined)+ service.activateGfsAuthScope = vi.fn()+ service.stopAllStreams = vi.fn()+ service.gfsUploadJobs = new Map([['upload-1', { job: { suspendForAuth: vi.fn() } }]])++ await expect(service.switchTeam('team-b')).resolves.toMatchObject({+ authenticated: true,+ me: { teamId: 'team-b' },+ })++ expect(service.suspendDesktopGfsUploadsForAuthBoundary).toHaveBeenCalledWith({+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ })+ expect(+ service.suspendDesktopGfsUploadsForAuthBoundary.mock.invocationCallOrder[0]+ ).toBeGreaterThan(service.authClient.switchTeam.mock.invocationCallOrder[0])+ })++ it('does not fence GFS uploads when an explicit switchTeam is rejected', async () => {+ const service = new AppService() as any+ service.sessionToken = 'team-a-token'+ service.me = {+ id: 'user-1',+ email: '[email protected]',+ name: 'Test User',+ picture: null,+ teamId: 'team-a',+ teamName: 'Team A',+ role: 'member',+ }+ service.gfsScopeIdentity = {+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ }+ service.authClient = {+ switchTeam: vi.fn().mockRejectedValue(new Error('team switch rejected')),+ }+ service.suspendDesktopGfsUploadsForAuthBoundary = vi.fn()++ await expect(service.switchTeam('team-b')).rejects.toThrow('team switch rejected')+ expect(service.suspendDesktopGfsUploadsForAuthBoundary).not.toHaveBeenCalled()+ })+it('uses the teamless session when a directly granted agent needs an RPC token', async () => {const service = new AppService() as anyservice.sessionToken = 'teamless-token'diff --git a/desktop-app/src/appService.ts b/desktop-app/src/appService.tsindex c456986f7..29d57184d 100644--- a/desktop-app/src/appService.ts+++ b/desktop-app/src/appService.ts@@ -298,7 +298,11 @@ const gfsUploadTransport: DesktopUploadTransport = {signal: withTimeout(signal, timeoutMs),} as RequestInit & { duplex: 'half' })const text = await response.text()- return { status: response.status, text }+ return {+ status: response.status,+ text,+ retryAfter: response.headers.get('retry-after') ?? undefined,+ }},}@@ -411,6 +415,8 @@ interface DesktopGfsUploadScope {authEpoch: number}+type DesktopGfsUploadIdentity = Omit<DesktopGfsUploadScope, 'drive' | 'authEpoch'>+interface PersistedDesktopGfsUpload {version: typeof DESKTOP_GFS_UPLOAD_STATE_VERSIONuploadId: string@@ -473,6 +479,15 @@ function normalizeDesktopUploadBaseUrl(value: string): string {return url.toString().replace(/\/+$/, '')}+function desktopGfsUploadIdentity(me: SessionMe): DesktopGfsUploadIdentity {+ return {+ ownerId: me.id,+ teamId: me.teamId ? String(me.teamId).trim() : null,+ environmentKey: getActiveEnvKey(),+ baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),+ }+}+function normalizeDesktopUploadDrive(value: string | undefined): string {const drive = String(value ?? 'main').trim()if (!drive) throw new Error('drive_required: GFS uploads require a canonical drive')@@ -803,6 +818,10 @@ export class AppService {private logoutInProgress = falseprivate gfsAuthEpoch = 0private gfsDispatchBlocked = true+ // A finite RPC may temporarily borrow another team session. Keep the+ // deliberate user-owned GFS scope stable across that hop so in-flight jobs+ // are not invalidated by the transient `me` value.+ private gfsScopeIdentity: DesktopGfsUploadIdentity | null = nullprivate gfsUploadStateQueue: Promise<void> = Promise.resolve()private readonly gfsUploadJobs = new Map<string,@@ -975,11 +994,12 @@ export class AppService {private async switchSessionToTeam(teamId: string, token = this.requireSessionToken()) {const targetTeamId = String(teamId || '').trim()if (!targetTeamId) throw new Error('teamId is required')- await this.suspendDesktopGfsUploadsForAuthBoundary()const switchGeneration = this.sessionGenerationif (this.sessionToken !== token || !this.me) {throw new Error('stale_auth_epoch: authenticated team scope changed before switch dispatch')}+ const previousIdentity =+ this.gfsScopeIdentity ?? (this.me ? desktopGfsUploadIdentity(this.me) : undefined)let switchedToken: string | null = nulltry {const switched = await this.authClient.switchTeam(token, targetTeamId)@@ -989,12 +1009,17 @@ export class AppService {}await this.commitSessionToken(switched.token, { refreshMe: true })if (!this.me) throw new Error('Team switch ended without an authenticated session')- this.activateGfsAuthScope()return switched.token} catch (error) {if (!switchedToken && this.sessionToken === token && this.me) {- this.activateGfsAuthScope()+ // A failed transient hop leaves the deliberate GFS scope untouched.} else {+ // Once the auth service has issued a replacement token, the previous+ // authenticated scope is no longer safe to continue. Fence and+ // persist its uploads before clearing the partially switched session;+ // otherwise a failed refresh/restore could leave a job running with a+ // token whose team ownership is unknown.+ await this.suspendDesktopGfsUploadsForAuthBoundary(previousIdentity)this.clearAuthenticatedSessionState()try {await this.tokenStore.clearSessionToken(getActiveEnvKey(), {@@ -1132,6 +1157,7 @@ export class AppService {this.sessionGeneration += 1this.gfsAuthEpoch += 1this.gfsDispatchBlocked = true+ this.gfsScopeIdentity = nullthis.stopAllStreams()this.sessionToken = nullthis.me = null@@ -1144,20 +1170,22 @@ export class AppService {unbindChatStore()}- private activateGfsAuthScope(): void {+ private activateGfsAuthScope(identityOverride?: DesktopGfsUploadIdentity): void {this.gfsAuthEpoch += 1this.gfsDispatchBlocked = false+ this.gfsScopeIdentity = identityOverride ?? (this.me ? desktopGfsUploadIdentity(this.me) : null)}private currentDesktopGfsUploadScope(driveValue: string | undefined): DesktopGfsUploadScope {if (this.gfsDispatchBlocked || !this.sessionToken || !this.me) {throw new Error('GFS upload dispatch is unavailable without an active authenticated scope')}+ const identity = this.gfsScopeIdentity+ if (!identity) {+ throw new Error('GFS upload dispatch is unavailable before scope activation')+ }return {- ownerId: this.me.id,- teamId: this.me.teamId ? String(this.me.teamId).trim() : null,- environmentKey: getActiveEnvKey(),- baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),+ ...identity,drive: normalizeDesktopUploadDrive(driveValue),authEpoch: this.gfsAuthEpoch,}@@ -1170,15 +1198,22 @@ export class AppService {}}- private async suspendDesktopGfsUploadsForAuthBoundary(): Promise<void> {- const oldIdentity = this.me- ? {- ownerId: this.me.id,- teamId: this.me.teamId ? String(this.me.teamId).trim() : null,- environmentKey: getActiveEnvKey(),- baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),- }- : null+ private async suspendDesktopGfsUploadsForAuthBoundary(identityOverride?: {+ ownerId: string+ teamId: string | null+ environmentKey: string+ baseUrl: string+ }): Promise<void> {+ const oldIdentity =+ identityOverride ??+ (this.me+ ? {+ ownerId: this.me.id,+ teamId: this.me.teamId ? String(this.me.teamId).trim() : null,+ environmentKey: getActiveEnvKey(),+ baseUrl: normalizeDesktopUploadBaseUrl(config.externalRestApiBaseUrl),+ }+ : null)this.gfsDispatchBlocked = truethis.gfsAuthEpoch += 1const jobs = new Set([@@ -2739,6 +2774,8 @@ export class AppService {async switchTeam(teamId: string): Promise<SessionState> {const targetTeamId = String(teamId || '').trim()if (!targetTeamId) throw new Error('teamId is required')+ const previousIdentity =+ this.gfsScopeIdentity ?? (this.me ? desktopGfsUploadIdentity(this.me) : undefined)// §4.5-3 (GAP-N4 main half): a deliberate team switch tears down every live// stream — the old team's progress/activity/status/notification sockets must// not survive it. Done AFTER the switch SUCCEEDS (silently), mirroring the@@ -2751,6 +2788,11 @@ export class AppService {// minting a cross-team RPC token to decide an approval) must NOT.await this.switchSessionToTeam(targetTeamId)if (!this.me) throw new Error('Team switch ended without an authenticated session')+ // A deliberate user switch is the only team-context boundary that fences+ // in-flight uploads. Transient runWithTeamContext hops intentionally use+ // switchSessionToTeam directly and must leave those jobs untouched.+ await this.suspendDesktopGfsUploadsForAuthBoundary(previousIdentity)+ this.activateGfsAuthScope()this.stopAllStreams()// Grants are keyed by userId, not by team, so they carry over — but every// cached org/agents/contexts answer is now about the wrong team. Drop thediff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.tsindex 6a0a205b3..8bf873a08 100644--- a/desktop-app/src/gfs/upload.test.ts+++ b/desktop-app/src/gfs/upload.test.ts@@ -151,6 +151,71 @@ describe('desktop GFS indexed uploader', () => {}})+ it('retries a lifecycle 429 using the transport Retry-After value', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-retry-after-'))+ try {+ const filePath = join(root, 'payload.bin')+ await writeFile(filePath, Buffer.from([1]))+ const session = {+ uploadId: '13131313-1313-4131-8131-131313131313',+ drive: 'main',+ operation: 'create' as const,+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+ let createCalls = 0+ const transport = {+ async requestJson<T>(method: 'GET' | 'POST', url: string): Promise<T> {+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))+ return enabledCapabilities() as T+ if (method === 'POST' && requestPath(url).endsWith('/uploads')) {+ createCalls += 1+ if (createCalls === 1)+ throw Object.assign(new Error('quota exceeded'), { status: 429, retryAfter: '0' })+ return { ok: true, data: session } as T+ }+ if (method === 'POST' && requestPath(url).endsWith('/complete'))+ return { ok: true, data: { ...session, state: 'completed' } } as T+ throw new Error(`unexpected ${method} ${url}`)+ },+ async requestPart(+ _url: string,+ _token: string,+ _headers: Record<string, string>,+ body: NodeJS.ReadableStream+ ) {+ for await (const _chunk of body as AsyncIterable<Buffer>) {+ /* drain */+ }+ return { status: 204, text: '' }+ },+ }++ await expect(+ new DesktopGfsUploadJob({+ baseUrl: 'https://api.example',+ token: 'token',+ filePath,+ name: 'payload.bin',+ drive: 'main',+ operation: 'create',+ parentRid: 'parent-retry-after',+ transport,+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(createCalls).toBe(2)+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })+it('resumes from sparse status and does not replay a committed part', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-resume-'))try {diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.tsindex 22a586986..223156274 100644--- a/desktop-app/src/gfs/upload.ts+++ b/desktop-app/src/gfs/upload.ts@@ -1,7 +1,7 @@import { createHash, randomUUID } from 'node:crypto'import { createReadStream } from 'node:fs'import { stat } from 'node:fs/promises'-import type { Readable } from 'node:stream'+import { type Readable, Transform } from 'node:stream'const GFS_UPLOAD_V2_PRODUCT_MAX_BYTES = 200 * 1024 * 1024const GFS_UPLOAD_V2_PREFERRED_PART_BYTES = 8 * 1024 * 1024@@ -11,6 +11,9 @@ const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3+export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3+export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000+const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250export interface DesktopUploadSession {uploadId: string@@ -71,6 +74,7 @@ interface Envelope<T> {interface UploadPartResponse {status: numbertext: string+ retryAfter?: string}interface UploadCapabilities {@@ -219,6 +223,8 @@ function retryableError(error: unknown): boolean {if (error instanceof Error && error.message.startsWith('stale_auth_epoch:')) return falseif (error instanceof Error && error.message.startsWith('upload_part_outcome_unknown:'))return false+ if (error instanceof Error && error.message.startsWith('upload_completion_outcome_unknown:'))+ return falseconst status =error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'? Number((error as { status: number }).status)@@ -228,8 +234,150 @@ function retryableError(error: unknown): boolean {return status === undefined || retryable(status)}-function sleep(ms: number): Promise<void> {- return new Promise(resolve => setTimeout(resolve, ms))+/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */+export function parseRetryAfter(+ value: string | null | undefined,+ nowMs = Date.now(),+ capMs = GFS_UPLOAD_RETRY_AFTER_CAP_MS+): number | undefined {+ if (value === null || value === undefined) return undefined+ const trimmed = value.trim()+ if (!trimmed) return undefined+ const seconds = Number(trimmed)+ let delayMs: number+ if (Number.isFinite(seconds) && seconds >= 0) {+ delayMs = seconds * 1_000+ } else {+ const dateMs = Date.parse(trimmed)+ if (!Number.isFinite(dateMs)) return undefined+ delayMs = Math.max(0, dateMs - nowMs)+ }+ if (!Number.isFinite(delayMs) || delayMs < 0) return undefined+ return Math.min(capMs, delayMs)+}++function retryAfterOf(error: unknown): number | undefined {+ if (!error || typeof error !== 'object') return undefined+ const value = error as {+ retryAfterMs?: unknown+ retryAfter?: unknown+ headers?: { get?: (name: string) => string | null }+ response?: { headers?: { get?: (name: string) => string | null } }+ bodyText?: unknown+ }+ if (typeof value.retryAfterMs === 'number' && Number.isFinite(value.retryAfterMs))+ return Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, Math.max(0, value.retryAfterMs))+ const header =+ typeof value.retryAfter === 'string'+ ? value.retryAfter+ : (value.headers?.get?.('retry-after') ?? value.response?.headers?.get?.('retry-after'))+ const parsedHeader = parseRetryAfter(typeof header === 'string' ? header : undefined)+ if (parsedHeader !== undefined) return parsedHeader+ if (typeof value.bodyText === 'string') {+ try {+ const parsed = JSON.parse(value.bodyText) as { retryAfterSeconds?: unknown }+ if (typeof parsed.retryAfterSeconds === 'number' && Number.isFinite(parsed.retryAfterSeconds))+ return Math.min(+ GFS_UPLOAD_RETRY_AFTER_CAP_MS,+ Math.max(0, parsed.retryAfterSeconds * 1_000)+ )+ } catch {+ // Preserve the transport's original error when its body is not JSON.+ }+ }+ return undefined+}++function ambiguousLifecycleError(error: unknown): boolean {+ const status =+ error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'+ ? Number((error as { status: number }).status)+ : undefined+ return status === undefined || status === 408 || (status !== undefined && status >= 500)+}++function terminalReconciliationError(error: unknown): boolean {+ if (!(error instanceof Error)) return false+ return [+ 'upload status changed during reconciliation',+ 'resume_part_mismatch:',+ 'source_changed:',+ 'upload_completion_outcome_unknown:',+ 'cannot reconcile ',+ ].some(prefix => error.message.startsWith(prefix))+}++interface LifecycleRecovery<T> {+ value: T+}++interface LifecycleRetryOptions<T> {+ signal?: AbortSignal+ attempts?: number+ reconcileOnError?: (error: unknown) => boolean+ reconcile?: (error: unknown) => Promise<LifecycleRecovery<T> | undefined>+}++/**+ * Bounded lifecycle retry primitive. It keeps Retry-After handling, abort+ * propagation, and state-reconciliation hooks identical across Desktop+ * create/pause/resume/complete/cancel/status requests.+ */+async function requestWithLifecycleRetries<T>(+ input: DesktopUploadInput,+ operation: () => Promise<T>,+ options: LifecycleRetryOptions<T> = {}+): Promise<T> {+ const attempts = Math.max(1, options.attempts ?? GFS_UPLOAD_RETRY_MAX_ATTEMPTS)+ let lastError: unknown+ for (let attempt = 0; attempt < attempts; attempt += 1) {+ assertAuthEpoch(input)+ try {+ return await operation()+ } catch (error) {+ lastError = error+ if (options.signal?.aborted) throw options.signal.reason ?? error+ const shouldReconcile = options.reconcile+ ? (options.reconcileOnError?.(error) ?? ambiguousLifecycleError(error))+ : false+ if (shouldReconcile) {+ try {+ const recovered = await options.reconcile!(error)+ if (recovered) return recovered.value+ } catch (reconcileError) {+ // A transient status/HEAD failure must not suppress the bounded+ // retry of the original request. Definitive geometry, auth, or+ // outcome-unknown errors remain terminal and are surfaced.+ if (terminalReconciliationError(reconcileError)) throw reconcileError+ }+ }+ if (!retryableError(error) || attempt + 1 >= attempts) throw error+ await sleep(+ retryAfterOf(error) ??+ Math.min(GFS_UPLOAD_RETRY_AFTER_CAP_MS, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ options.signal+ )+ }+ }+ throw lastError+}++function sleep(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve, reject) => {+ if (signal?.aborted) {+ reject(signal.reason)+ return+ }+ const timer = setTimeout(resolve, ms)+ signal?.addEventListener(+ 'abort',+ () => {+ clearTimeout(timer)+ reject(signal.reason)+ },+ { once: true }+ )+ })}interface DesktopUploadPartChecksum {@@ -349,16 +497,22 @@ async function loadUploadStatus(partCount?: number},signal?: AbortSignal,- timeoutMs = UPLOAD_TIMEOUT_MS+ timeoutMs = UPLOAD_TIMEOUT_MS,+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS): Promise<DesktopUploadStatus> {assertAuthEpoch(input)- await input.transport.requestJson<Envelope<undefined>>(- 'HEAD',- joinUrl(- input.baseUrl,- uploadPath(expected.drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}`)- ),- { token: input.token, timeoutMs, signal }+ await requestWithLifecycleRetries(+ input,+ () =>+ input.transport.requestJson<Envelope<undefined>>(+ 'HEAD',+ joinUrl(+ input.baseUrl,+ uploadPath(expected.drive, `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}`)+ ),+ { token: input.token, timeoutMs, signal }+ ),+ { signal, attempts: retryAttempts })let cursor: string | undefined@@ -369,17 +523,22 @@ async function loadUploadStatus(const query = new URLSearchParams({ limit: '256' })if (cursor) query.set('cursor', cursor)const status = unwrap<DesktopUploadStatus>(- await input.transport.requestJson<Envelope<DesktopUploadStatus>>(- 'GET',- joinUrl(- input.baseUrl,- uploadPath(- expected.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}/status`,- query- )- ),- { token: input.token, timeoutMs, signal }+ await requestWithLifecycleRetries(+ input,+ () =>+ input.transport.requestJson<Envelope<DesktopUploadStatus>>(+ 'GET',+ joinUrl(+ input.baseUrl,+ uploadPath(+ expected.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(uploadId)}/status`,+ query+ )+ ),+ { token: input.token, timeoutMs, signal }+ ),+ { signal, attempts: retryAttempts }))session ??= status.session@@ -440,14 +599,18 @@ async function reconcileAmbiguousPart(partCount: session.partCount,},signal,- GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,+ 1)} catch (error) {if (signal?.aborted) throw signal.reason ?? errorlastError = errorif (!retryableError(error) || attempt + 1 >= GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)throw partOutcomeUnknown(partNumber, error)- await sleep(250 * 2 ** attempt)+ await sleep(+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ signal+ )continue}@@ -485,6 +648,7 @@ async function putPart(start: number,length: number,signal?: AbortSignal,+ onByteProgress?: (loadedBytes: number) => void,onRetryableFailure?: () => void,onPartSuccess?: () => void): Promise<void> {@@ -494,11 +658,21 @@ async function putPart(await assertSourceIdentity(input)const checksum = await checksumPart(input.filePath, start, start + length - 1)await assertSourceIdentity(input)- const body = createReadStream(input.filePath, {+ const source = createReadStream(input.filePath, {start,end: start + length - 1,highWaterMark: 1024 * 1024,})+ let loadedBytes = 0+ const body = new Transform({+ transform(chunk, _encoding, callback) {+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)+ loadedBytes += buffer.length+ onByteProgress?.(Math.min(length, loadedBytes))+ callback(null, buffer)+ },+ })+ source.pipe(body)let response: UploadPartResponsetry {assertAuthEpoch(input)@@ -541,7 +715,10 @@ async function putPart(return}if (attempt >= 2) throw error- await sleep(250 * 2 ** attempt)+ await sleep(+ retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ signal+ )attempt += 1continue}@@ -569,7 +746,10 @@ async function putPart(}}if (attempt >= 2) throw uploadPartError(response.status, response.text)- await sleep(250 * 2 ** attempt)+ await sleep(+ retryAfterOf(response) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),+ signal+ )attempt += 1}}@@ -595,8 +775,10 @@ async function runParts(let retryableFailures = 0let consecutiveRetryableFailures = 0let downgraded = false+ const inFlightLoaded = new Map<number, number>()const advertisedConcurrency = Number(input.advertisedConcurrency)const concurrency = Math.min(+ GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,Number.isSafeInteger(advertisedConcurrency) && advertisedConcurrency > 0? advertisedConcurrency: GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,@@ -621,6 +803,21 @@ async function runParts(const registerPartSuccess = (): void => {consecutiveRetryableFailures = 0}+ const calculateUploadedBytes = (): number => {+ const committedBytes = [...completedParts].reduce(+ (sum, number) =>+ sum + Math.min(session.partBytes, session.expectedBytes - number * session.partBytes),+ 0+ )+ const inFlightBytes = [...inFlightLoaded.entries()].reduce(+ (sum, [number, loaded]) => (completedParts.has(number) ? sum : sum + loaded),+ 0+ )+ const observed = Math.min(session.expectedBytes, committedBytes + inFlightBytes)+ // Part bytes are committed before the final resource publication. Do not+ // expose a visually complete bar until the complete receipt is durable.+ return Math.min(Math.max(0, session.expectedBytes - 1), observed)+ }const execute = async (work: typeof parts, workers: number): Promise<void> => {let next = 0const runWorker = async (workerIndex: number): Promise<void> => {@@ -641,19 +838,24 @@ async function runParts(part.offsetBytes,part.lengthBytes,signal,+ loaded => {+ inFlightLoaded.set(+ part.partNumber,+ Math.max(inFlightLoaded.get(part.partNumber) ?? 0, loaded)+ )+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)+ },registerRetryableFailure,registerPartSuccess)completedParts.add(part.partNumber)+ inFlightLoaded.delete(part.partNumber)failed.delete(part.partNumber)retryableFailed.delete(part.partNumber)- const completedBytes = [...completedParts].reduce(- (sum, number) =>- sum + Math.min(session.partBytes, session.expectedBytes - number * session.partBytes),- 0- )- input.onProgress?.(completedBytes, session.expectedBytes)+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)} catch (error) {+ inFlightLoaded.delete(part.partNumber)+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)if (shouldPause()) returnif (signal?.aborted ||@@ -684,6 +886,11 @@ function uploadPartError(status: number, text: string): Error & { status: numberreturn error}+function errorCodeOf(error: unknown): string | undefined {+ const value = error as { code?: unknown } | null+ return value && typeof value.code === 'string' ? value.code : undefined+}+export class DesktopGfsUploadJob {private readonly abortController = new AbortController()private readonly authBoundaryController = new AbortController()@@ -746,23 +953,31 @@ export class DesktopGfsUploadJob {throw new Error(`cannot pause upload in ${session.state}`)this.pauseRequested = truethis.assertAuthEpoch()- const paused = unwrap<DesktopUploadSession>(- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(- 'POST',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/pause`+ const paused = await requestWithLifecycleRetries(+ this.input,+ async () =>+ unwrap<DesktopUploadSession>(+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(+ 'POST',+ joinUrl(+ this.input.baseUrl,+ uploadPath(+ this.input.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/pause`+ )+ ),+ {+ token: this.input.token,+ body: {},+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.authBoundaryController.signal,+ })),- {- token: this.input.token,- body: {},- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.authBoundaryController.signal,- }- )+ {+ signal: this.authBoundaryController.signal,+ reconcile: () => this.reconcileMutation(session, 'pause'),+ })this.session = pausedthis.state = 'paused'@@ -781,23 +996,31 @@ export class DesktopGfsUploadJob {if (session.state !== 'paused') return this.start()this.pauseRequested = falsethis.assertAuthEpoch()- const resumed = unwrap<DesktopUploadSession>(- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(- 'POST',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/resume`+ const resumed = await requestWithLifecycleRetries(+ this.input,+ async () =>+ unwrap<DesktopUploadSession>(+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(+ 'POST',+ joinUrl(+ this.input.baseUrl,+ uploadPath(+ this.input.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/resume`+ )+ ),+ {+ token: this.input.token,+ body: {},+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.authBoundaryController.signal,+ })),- {- token: this.input.token,- body: {},- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.authBoundaryController.signal,- }- )+ {+ signal: this.authBoundaryController.signal,+ reconcile: () => this.reconcileMutation(session, 'resume'),+ })this.session = resumedthis.state = 'uploading'@@ -818,65 +1041,68 @@ export class DesktopGfsUploadJob {this.abortController.abort(new Error('GFS upload canceled'))const session = this.sessionif (session && !['completed', 'aborted'].includes(session.state)) {+ let reconciledCompleted = falsetry {- this.assertAuthEpoch()- await this.input.transport.requestJson(- 'DELETE',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}`- )- ),+ await requestWithLifecycleRetries(+ this.input,+ () =>+ this.input.transport+ .requestJson(+ 'DELETE',+ joinUrl(+ this.input.baseUrl,+ uploadPath(+ this.input.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}`+ )+ ),+ {+ token: this.input.token,+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.authBoundaryController.signal,+ }+ )+ .then(() => undefined),{- token: this.input.token,- timeoutMs: UPLOAD_TIMEOUT_MS,signal: this.authBoundaryController.signal,+ reconcileOnError: error =>+ ambiguousLifecycleError(error) ||+ Boolean(+ error && typeof error === 'object' && (error as { status?: unknown }).status === 409+ ) ||+ errorCodeOf(error) === 'upload_finalizing',+ reconcile: async error => {+ const observed = await this.observeSession(+ session,+ GFS_UPLOAD_RETRY_MAX_ATTEMPTS,+ this.authBoundaryController.signal+ )+ this.session = observed+ if (observed.state === 'completed') {+ reconciledCompleted = true+ return { value: undefined }+ }+ if (observed.state === 'aborted') return { value: undefined }+ if (observed.state === 'finalizing') throw error+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined+ throw new Error(`cannot reconcile cancel upload in ${observed.state}`)+ },})} catch (error) {- const status =- error &&- typeof error === 'object' &&- typeof (error as { status?: unknown }).status === 'number'- ? Number((error as { status: number }).status)- : undefined- if (status === 409) {- this.assertAuthEpoch()- const observed = unwrap<DesktopUploadStatus>(- await this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(- 'GET',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(session.uploadId)}/status`,- new URLSearchParams({ limit: '256' })- )- ),- {- token: this.input.token,- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.authBoundaryController.signal,- }- )- )- this.session = observed.session- if (observed.session.state === 'completed') {- this.canceled = false- this.state = 'completed'- this.uploadedBytes = observed.session.expectedBytes- this.emit()- await this.input.onClearPersisted?.(session.uploadId)- return- }- }this.canceled = falsethis.state = 'failed'this.emit()throw error}+ if (reconciledCompleted) {+ this.canceled = false+ this.state = 'completed'+ this.uploadedBytes = this.session?.expectedBytes ?? session.expectedBytes+ this.emit()+ await this.input.onClearPersisted?.(session.uploadId)+ return+ }}await this.runPromise?.catch(() => undefined)this.state = 'aborted'@@ -939,6 +1165,70 @@ export class DesktopGfsUploadJob {}}+ private async observeSession(+ session: DesktopUploadSession,+ retryAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS,+ signal: AbortSignal | undefined = this.authBoundaryController.signal+ ): Promise<DesktopUploadSession> {+ return (+ await loadUploadStatus(+ this.input,+ session.uploadId,+ {+ drive: session.drive,+ expectedBytes: session.expectedBytes,+ partBytes: session.partBytes,+ partCount: session.partCount,+ },+ signal,+ GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS,+ retryAttempts+ )+ ).session+ }++ private async reconcileMutation(+ session: DesktopUploadSession,+ action: 'pause' | 'resume'+ ): Promise<LifecycleRecovery<DesktopUploadSession> | undefined> {+ const observed = await this.observeSession(session)+ if (+ (action === 'pause' && observed.state === 'paused') ||+ (action === 'resume' && ['initiated', 'uploading', 'completed'].includes(observed.state))+ ) {+ return { value: observed }+ }+ if (+ (action === 'pause' && ['initiated', 'uploading'].includes(observed.state)) ||+ (action === 'resume' && observed.state === 'paused')+ ) {+ return undefined+ }+ throw new Error(`cannot reconcile ${action} upload in ${observed.state}`)+ }++ private async reconcileCompletion(): Promise<+ LifecycleRecovery<DesktopUploadSession> | undefined+ > {+ const session = this.session+ if (!session) throw new Error('GFS upload session is unavailable during completion')+ for (let attempt = 0; attempt < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS; attempt += 1) {+ const observed = await this.observeSession(session, 1)+ if (observed.state === 'completed') return { value: observed }+ if (['initiated', 'uploading', 'paused'].includes(observed.state)) return undefined+ if (observed.state !== 'finalizing')+ throw new Error(`cannot reconcile upload completion in ${observed.state}`)+ if (attempt + 1 < GFS_UPLOAD_V2_RECONCILE_ATTEMPTS)+ await sleep(+ GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt,+ this.authBoundaryController.signal+ )+ }+ throw new Error(+ 'upload_completion_outcome_unknown: finalization is still in progress; resume the persisted session'+ )+ }+private async prepare(): Promise<{session: DesktopUploadSessioncommitted: Set<number>@@ -959,14 +1249,19 @@ export class DesktopGfsUploadJob {// receipts use the { ok, data, error } envelope; accepting an envelope// here would hide drift between Desktop and the writer's public contract.capabilities = parseUploadCapabilities(- await this.input.transport.requestJson<UploadCapabilities>(- 'GET',- joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/capabilities')),- {- token: this.input.token,- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.abortController.signal,- }+ await requestWithLifecycleRetries(+ this.input,+ () =>+ this.input.transport.requestJson<UploadCapabilities>(+ 'GET',+ joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/capabilities')),+ {+ token: this.input.token,+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.abortController.signal,+ }+ ),+ { signal: this.abortController.signal }))} catch (error) {@@ -979,8 +1274,14 @@ export class DesktopGfsUploadJob {throw new DesktopUploadCapabilityError('Resumable GFS uploads are not enabled on this writer.')- if (typeof resumable.maxFileBytes === 'number' && file.size > resumable.maxFileBytes)- throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)+ if (resumable.maxFileBytes !== undefined) {+ if (!Number.isSafeInteger(resumable.maxFileBytes) || resumable.maxFileBytes < 0)+ throw new DesktopUploadCapabilityError(+ 'GFS resumable capabilities advertised an invalid file ceiling'+ )+ if (file.size > resumable.maxFileBytes)+ throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)+ }this.input.advertisedConcurrency = resumable.maxConcurrentPartsPerSessionthis.input.fallbackConcurrency = resumable.fallbackConcurrencyconst resumeId = this.session?.uploadId ?? this.input.resumeUploadId@@ -1001,24 +1302,30 @@ export class DesktopGfsUploadJob {const target = this.input.operation === 'create' ? this.input.parentRid : this.input.resourceRidif (!target) throw new Error(`${this.input.operation} upload target is required`)this.assertAuthEpoch()+ const createBody = {+ operation: this.input.operation,+ drive,+ sizeBytes: file.size,+ idempotencyKey: randomUUID(),+ ...(this.input.operation === 'create'+ ? { parentRid: target, name: this.input.name }+ : { resourceRid: target, ifMatch: this.input.ifMatch }),+ }const created = unwrap<DesktopUploadSession>(- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(- 'POST',- joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/uploads')),- {- token: this.input.token,- body: {- operation: this.input.operation,- drive,- sizeBytes: file.size,- idempotencyKey: randomUUID(),- ...(this.input.operation === 'create'- ? { parentRid: target, name: this.input.name }- : { resourceRid: target, ifMatch: this.input.ifMatch }),- },- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.abortController.signal,- }+ await requestWithLifecycleRetries(+ this.input,+ () =>+ this.input.transport.requestJson<Envelope<DesktopUploadSession>>(+ 'POST',+ joinUrl(this.input.baseUrl, uploadPath(drive, '/api/v1/me/gfs/uploads')),+ {+ token: this.input.token,+ body: createBody,+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.abortController.signal,+ }+ ),+ { signal: this.abortController.signal }))if (created.drive !== drive) throw new Error('upload_drive_mismatch: writer changed drive')@@ -1080,21 +1387,26 @@ export class DesktopGfsUploadJob {this.state = 'paused'this.assertAuthEpoch()const status = unwrap<DesktopUploadStatus>(- await this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(- 'GET',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,- new URLSearchParams({ limit: '256' })- )- ),- {- token: this.input.token,- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.authBoundaryController.signal,- }+ await requestWithLifecycleRetries(+ this.input,+ () =>+ this.input.transport.requestJson<Envelope<DesktopUploadStatus>>(+ 'GET',+ joinUrl(+ this.input.baseUrl,+ uploadPath(+ this.input.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/status`,+ new URLSearchParams({ limit: '256' })+ )+ ),+ {+ token: this.input.token,+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.authBoundaryController.signal,+ }+ ),+ { signal: this.authBoundaryController.signal }))this.session = status.session@@ -1108,23 +1420,33 @@ export class DesktopGfsUploadJob {this.assertAuthEpoch()this.state = 'finalizing'this.emit()- const completed = unwrap<DesktopUploadSession>(- await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(- 'POST',- joinUrl(- this.input.baseUrl,- uploadPath(- this.input.drive,- `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`+ const completed = await requestWithLifecycleRetries<DesktopUploadSession>(+ this.input,+ async () =>+ unwrap<DesktopUploadSession>(+ await this.input.transport.requestJson<Envelope<DesktopUploadSession>>(+ 'POST',+ joinUrl(+ this.input.baseUrl,+ uploadPath(+ this.input.drive,+ `/api/v1/me/gfs/uploads/${encodeURIComponent(prepared.session.uploadId)}/complete`+ )+ ),+ {+ token: this.input.token,+ body: {},+ timeoutMs: UPLOAD_TIMEOUT_MS,+ signal: this.abortController.signal,+ })),- {- token: this.input.token,- body: {},- timeoutMs: UPLOAD_TIMEOUT_MS,- signal: this.abortController.signal,- }- )+ {+ signal: this.abortController.signal,+ reconcileOnError: error =>+ ambiguousLifecycleError(error) || errorCodeOf(error) === 'upload_finalizing',+ reconcile: () => this.reconcileCompletion(),+ })if (completed.drive !== canonicalDrive(this.input.drive))throw new Error('upload_drive_mismatch: completion changed drive')diff --git a/desktop-app/src/httpClient.ts b/desktop-app/src/httpClient.tsindex 5ab838154..3013b5fae 100644--- a/desktop-app/src/httpClient.ts+++ b/desktop-app/src/httpClient.ts@@ -3,11 +3,13 @@ import { config } from './config.js'export class ApiError extends Error {status: numberbodyText: string+ retryAfter?: string- constructor(message: string, status: number, bodyText: string) {+ constructor(message: string, status: number, bodyText: string, retryAfter?: string | null) {super(message)this.status = statusthis.bodyText = bodyText+ if (retryAfter) this.retryAfter = retryAfter}}@@ -125,7 +127,12 @@ export async function requestJson<T>(} catch {// Keep raw text as message.}- throw new ApiError(`${response.status} ${response.statusText}: ${msg}`, response.status, raw)+ throw new ApiError(+ `${response.status} ${response.statusText}: ${msg}`,+ response.status,+ raw,+ response.headers.get('retry-after')+ )}if (!raw) return {} as Tdiff --git a/desktop-app/test/httpClient.test.ts b/desktop-app/test/httpClient.test.tsindex 611cb47b0..d1cd695dd 100644--- a/desktop-app/test/httpClient.test.ts+++ b/desktop-app/test/httpClient.test.ts@@ -16,12 +16,13 @@ function okResponse(body?: unknown): Response {} as unknown as Response}-function errorResponse(status: number, body: string): Response {+function errorResponse(status: number, body: string, headers?: Record<string, string>): Response {return {ok: false,status,statusText: 'Error',text: () => Promise.resolve(body),+ headers: new Headers(headers),} as unknown as Response}@@ -91,4 +92,16 @@ describe('requestJson — transient retry', () => {expect(fetchSpy).toHaveBeenCalledTimes(1)})++ it('preserves Retry-After on HTTP errors for bounded upload lifecycle retries', async () => {+ const fetchSpy = vi.mocked(global.fetch)+ fetchSpy.mockResolvedValueOnce(+ errorResponse(429, '{"error":"quota_exceeded"}', { 'retry-after': '7' })+ )++ await expect(requestJson('POST', 'http://localhost/upload')).rejects.toMatchObject({+ status: 429,+ retryAfter: '7',+ })+ })})diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex 6447f3a0b..c589ae108 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.68",+ "version": "0.1.69","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.68",+ "version": "0.1.69","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex f21f91fb9..b98e1c008 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.68",+ "version": "0.1.69","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/routes/gfs.ts b/external-rest-api/src/routes/gfs.tsindex cd1c53063..08779f96e 100644--- a/external-rest-api/src/routes/gfs.ts+++ b/external-rest-api/src/routes/gfs.ts@@ -33,6 +33,7 @@ const STREAM_HEADERS = ['content-length','content-disposition','cache-control',+ 'location','upload-offset','upload-length','upload-part-bytes',@@ -44,14 +45,19 @@ const STREAM_HEADERS = ['upload-part-offset','upload-part-length','upload-checksum',+ 'retry-after',+ 'x-ratelimit-limit',+ 'x-ratelimit-remaining',]function forwardControlApiError(error: unknown, res: Response, next: NextFunction): void {if (error instanceof ControlApiError && PROPAGATED.has(error.status)) {const body =error.body && typeof error.body === 'object' ? error.body : { error: String(error.message) }- const retryAfter = error.headers?.get('retry-after')- if (retryAfter) res.setHeader('Retry-After', retryAfter)+ for (const header of STREAM_HEADERS) {+ const value = error.headers?.get(header)+ if (value) res.setHeader(header, value)+ }res.status(error.status).json(body)return}diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.tsindex 3b5f06e1a..5e8160092 100644--- a/gfs-controller/src/api/serve.test.ts+++ b/gfs-controller/src/api/serve.test.ts@@ -708,6 +708,27 @@ describe("GfsServingHandler — write routes (governed mutation)", () => {expect(calls).toHaveLength(0); // never reached the write service});+ it.each([-1, 1.5, "2"])(+ "400 path_invalid for an invalid If-Match value before authorization: %j",+ async (ifMatch) => {+ const res = new FakeRes();+ const { d, calls, authzOps } = writeDeps();+ await run(+ d,+ reqBody(`/v1/resources/${RID}/content`, {+ method: "PUT",+ auth: "Bearer t",+ body: { content: "new", ifMatch },+ }),+ res+ );+ expect(res.statusCode).toBe(400);+ expect((res.json as { error: { code: string } }).error.code).toBe("path_invalid");+ expect(authzOps).toEqual([]);+ expect(calls).toHaveLength(0);+ }+ );+it("a NON-agent (user) replace WITHOUT If-Match is allowed (agent invariant only)", async () => {const res = new FakeRes();const { d, calls } = writeDeps({ verifyToken: () => USER_WRITE_CLAIMS });