Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(gfs): close upload review gaps and runtime gates
Alfredo Lopez893011ea8226
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex e795575b3..0d9088665 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -428,6 +428,9 @@ jobs:- name: Test minikube GFS Upload v2 profile contractrun: bash scripts/tests/test-minikube-gfs-upload-v2-profile.sh+ - name: Test GFS Upload v2 cross-client geometry contract+ run: bash scripts/tests/test-gfs-upload-v2-geometry-contract.sh+# The NetworkPolicy probe runs on an E2E-only fixture that the default# (pulled) setup never acquires. This proves the target says so up front# instead of timing out on ErrImageNeverPull, and that it still buildsdiff --git a/control-api/src/app.ts b/control-api/src/app.tsindex 67aa412c0..fff8ffbde 100644--- a/control-api/src/app.ts+++ b/control-api/src/app.ts@@ -63,7 +63,7 @@ const RPC_HOST_ACCESS_PATH = /^\/api\/v1\/rpc\/access\/users\/[^/]+\/mcp-hosts\/// proxy untouched; parsing them as JSON would either reject the first binary// byte or buffer the whole part in the control plane.const GFS_UPLOAD_PART_PATH =- /^\/api\/v1\/(?:gfs\/proxy\/v1|external\/gfs)\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i+ /^\/api\/v1\/(?:gfs\/proxy\/v1|external\/gfs)\/uploads\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/parts\/[0-9]+$/iexport function createApp(gateway: K8sGateway) {const traceIngestDb: DbClient = meterTracingDbClient({diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.tsindex d5fa6c032..530f87552 100644--- a/control-api/src/middleware/gfsUploadAdmission.ts+++ b/control-api/src/middleware/gfsUploadAdmission.ts@@ -65,7 +65,14 @@ export function createObservedGfsUploadPartBody(}function sourceIp(req: ExternalAuthedRequest): string {- const forwarded = String(req.headers['x-gfs-upload-source-ip'] || '').trim()+ // Only the authenticated external-rest-api service may assert the original+ // edge address. Direct callers of Control API (including other valid+ // service identities) must use the socket/proxy-derived address so they+ // cannot rotate an arbitrary forwarded header to evade the IP bucket.+ const forwarded =+ req.internalService?.name === 'external-rest-api'+ ? String(req.headers['x-gfs-upload-source-ip'] || '').trim()+ : ''if (forwarded && forwarded.length <= 64 && isIP(forwarded)) return forwardedconst direct = String(req.ip || req.socket.remoteAddress || 'unknown').trim()return direct.length <= 64 ? direct : 'unknown'@@ -132,7 +139,7 @@ export function gfsUploadAdmission(if (isPart) {const parsed = declaredGfsUploadPartBytes(req)if (!parsed.ok) {- gfsUploadAdmissionBytesTotal.inc({ result: 'rejected' }, 0)+ gfsUploadAdmissionRequestsTotal.inc({ limit: 'part_body', result: 'rejected' }, 1)res.status(parsed.status).json({ error: parsed.error })return}diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.tsindex adc573f95..45e2f00ec 100644--- a/control-api/src/routes/gfs/proxy.ts+++ b/control-api/src/routes/gfs/proxy.ts@@ -29,7 +29,8 @@ const PASSTHROUGH_RESPONSE_HEADERS = ['upload-checksum','retry-after',]-const UPLOAD_PART_PATH = /^\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+(?:\?|$)/i+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'+const UPLOAD_PART_PATH = new RegExp(`^/v1/uploads/${UUID_PATH}/parts/[0-9]+(?:\\?|$)`, 'i')const UPLOAD_PATH = /^\/v1\/(?:capabilities|uploads)(?:\/|\?|$)/iconst STREAM_REQUEST_HEADERS = ['content-type',diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex d5d8ce30c..f7e44cf3d 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -139,11 +139,29 @@ export type RateLimitConcurrencyLease = {release: () => Promise<void>}+export type RateLimitConcurrencyLeaseOptions = {+ /** Test/integration seam; production callers use the dedicated pool client. */+ client?: PoolClient+}+let concurrencyClient: PoolClient | null = nulllet concurrencyClientPromise: Promise<PoolClient> | null = null-const heldConcurrencySlots = new Set<string>()+// Session-scoped advisory locks are re-entrant only on the same PostgreSQL+// connection. Keep the local guard per client so an integration test (and any+// future multi-pool caller) still exercises the database lock across sessions.+const heldConcurrencySlotsByClient = new WeakMap<object, Set<string>>()let concurrencyOperation = Promise.resolve()+function heldSlotsFor(client: PoolClient): Set<string> {+ const key = client as unknown as object+ let slots = heldConcurrencySlotsByClient.get(key)+ if (!slots) {+ slots = new Set<string>()+ heldConcurrencySlotsByClient.set(key, slots)+ }+ return slots+}+async function serializeConcurrencyOperation<T>(operation: () => Promise<T>): Promise<T> {const previous = concurrencyOperationlet finish!: () => void@@ -158,8 +176,14 @@ async function serializeConcurrencyOperation<T>(operation: () => Promise<T>): Pr}}-function advisoryKey(value: string): number {- return createHash('sha256').update(value).digest().readInt32BE(0)+type AdvisoryLockKey = {+ high: number+ low: number+}++function advisoryKey(value: string): AdvisoryLockKey {+ const digest = createHash('sha256').update(value).digest()+ return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }}async function getConcurrencyClient(): Promise<PoolClient> {@@ -173,7 +197,7 @@ async function getConcurrencyClient(): Promise<PoolClient> {'rate limiter concurrency connection failed')if (concurrencyClient === client) concurrencyClient = null- heldConcurrencySlots.clear()+ heldSlotsFor(client).clear()client.release(true)})return client@@ -188,16 +212,21 @@ async function getConcurrencyClient(): Promise<PoolClient> {async function unlockSlots(client: PoolClient,- slots: Array<{ key: number; slot: number; identity: string }>-): Promise<void> {+ slots: Array<{ key: AdvisoryLockKey; identity: string }>+): Promise<boolean> {+ const heldSlots = heldSlotsFor(client)+ let success = truefor (const acquired of slots.reverse()) {- heldConcurrencySlots.delete(acquired.identity)try {await client.query('SELECT pg_advisory_unlock($1::integer, $2::integer)', [- acquired.key,- acquired.slot,+ acquired.key.high,+ acquired.key.low,])+ heldSlots.delete(acquired.identity)} catch (error) {+ // The lock state is unknown after an unlock error. Keep the local guard+ // until the connection error path invalidates this client; otherwise a+ // same-session reentrant pg_try_advisory_lock could admit a duplicate.rootLogger.warn({event: 'rate_limit_concurrency_unlock_error',@@ -205,8 +234,20 @@ async function unlockSlots(},'rate limiter concurrency unlock failed')+ success = false+ break}}+ return success+}++async function invalidateDedicatedConcurrencyClient(client: PoolClient): Promise<void> {+ if (concurrencyClient !== client) return+ concurrencyClient = null+ heldSlotsFor(client).clear()+ // Advisory locks are session-scoped. Destroying this dedicated client is the+ // only fail-closed way to release a lock whose unlock response is unknown.+ client.release(true)}/**@@ -216,7 +257,8 @@ async function unlockSlots(* reentrancy from admitting the same slot twice.*/export async function acquireRateLimitConcurrencyLease(- requirements: readonly RateLimitConcurrencyRequirement[]+ requirements: readonly RateLimitConcurrencyRequirement[],+ options: RateLimitConcurrencyLeaseOptions = {}): Promise<RateLimitConcurrencyLease> {for (const requirement of requirements) {if (!Number.isSafeInteger(requirement.maxConcurrent) || requirement.maxConcurrent < 1)@@ -225,7 +267,7 @@ export async function acquireRateLimitConcurrencyLease(return serializeConcurrencyOperation(async () => {let client: PoolClienttry {- client = await getConcurrencyClient()+ client = options.client ?? (await getConcurrencyClient())} catch (error) {rootLogger.warn({@@ -237,32 +279,39 @@ export async function acquireRateLimitConcurrencyLease(return { allowed: false, backendAvailable: false, release: async () => undefined }}- const acquired: Array<{ key: number; slot: number; identity: string }> = []+ const acquired: Array<{ key: AdvisoryLockKey; identity: string }> = []try {for (const requirement of requirements) {- const key = advisoryKey(requirement.bucketKey)- let selected: { key: number; slot: number; identity: string } | null = null+ const bucketKey = advisoryKey(requirement.bucketKey)+ let selected: { key: AdvisoryLockKey; identity: string } | null = nullfor (let slot = 0; slot < requirement.maxConcurrent; slot += 1) {- const identity = `${key}:${slot}`- if (heldConcurrencySlots.has(identity)) continue+ // Keep both halves of the SHA-256-derived key in the PostgreSQL+ // advisory lock identity. The slot is folded into the low half so+ // distinct buckets do not collide merely because their first 32+ // digest bits match.+ const key = { high: bucketKey.high, low: bucketKey.low ^ slot }+ const identity = `${key.high}:${key.low}`+ if (heldSlotsFor(client).has(identity)) continueconst result = await client.query<{ acquired: boolean }>('SELECT pg_try_advisory_lock($1::integer, $2::integer) AS acquired',- [key, slot]+ [key.high, key.low])if (result.rows[0]?.acquired === true) {- selected = { key, slot, identity }- heldConcurrencySlots.add(identity)+ selected = { key, identity }+ heldSlotsFor(client).add(identity)acquired.push(selected)break}}if (!selected) {- await unlockSlots(client, acquired)+ const unlocked = await unlockSlots(client, acquired)+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)return { allowed: false, backendAvailable: true, release: async () => undefined }}}} catch (error) {- await unlockSlots(client, acquired)+ const unlocked = await unlockSlots(client, acquired)+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)rootLogger.warn({event: 'rate_limit_concurrency_db_error',@@ -280,7 +329,10 @@ export async function acquireRateLimitConcurrencyLease(release: async () => {if (released) returnreleased = true- await serializeConcurrencyOperation(() => unlockSlots(client, acquired))+ await serializeConcurrencyOperation(async () => {+ const unlocked = await unlockSlots(client, acquired)+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)+ })},}})diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.tsindex b8b5e5375..93cd1c50c 100644--- a/control-api/test/gfsUploadAdmission.test.ts+++ b/control-api/test/gfsUploadAdmission.test.ts@@ -7,12 +7,26 @@ import {declaredGfsUploadPartBytes,gfsUploadAdmission,} from '../src/middleware/gfsUploadAdmission.js'+import {+ requireInternalService,+ requireInternalToken,+} from '../src/middleware/internalServiceAuth.js'-const { acquireMock, checkMock, metricDec, metricInc, releaseMock } = vi.hoisted(() => ({+const {+ acquireMock,+ checkMock,+ metricActiveInc,+ metricBytesInc,+ metricDec,+ metricRequestsInc,+ releaseMock,+} = vi.hoisted(() => ({acquireMock: vi.fn(),checkMock: vi.fn(),+ metricActiveInc: vi.fn(),+ metricBytesInc: vi.fn(),metricDec: vi.fn(),- metricInc: vi.fn(),+ metricRequestsInc: vi.fn(),releaseMock: vi.fn(),}))@@ -27,6 +41,10 @@ vi.mock('../src/config.js', () => ({gfsUploadMaxActivePerIp: 16,gfsUploadMaxActiveGlobal: 32,gfsUploadMaxPartBytes: 16 * 1024 * 1024,+ internalServiceTokens: {+ 'external-rest-api': 'dev-external-rest-api-token',+ 'rpc-proxy': 'dev-rpc-proxy-token',+ },},}))@@ -36,9 +54,9 @@ vi.mock('../src/services/rateLimiterService.js', () => ({}))vi.mock('../src/observability/metrics.js', () => ({- gfsUploadAdmissionActiveRequests: { inc: metricInc, dec: metricDec },- gfsUploadAdmissionBytesTotal: { inc: metricInc },- gfsUploadAdmissionRequestsTotal: { inc: metricInc },+ gfsUploadAdmissionActiveRequests: { inc: metricActiveInc, dec: metricDec },+ gfsUploadAdmissionBytesTotal: { inc: metricBytesInc },+ gfsUploadAdmissionRequestsTotal: { inc: metricRequestsInc },}))function rateCheck(@@ -62,7 +80,7 @@ function rateCheck(}}-function buildPartApp() {+function buildPartApp(serviceName?: string) {const app = express()app.put('/external/gfs/uploads/:id/parts/:part',@@ -70,6 +88,7 @@ function buildPartApp() {;(req as typeof req & { externalAuth: { userId: string } }).externalAuth = {userId: 'user-1',}+ if (serviceName) req.internalService = { name: serviceName }next()},gfsUploadAdmission,@@ -97,11 +116,34 @@ function buildLifecycleApp() {return app}+function buildAuthenticatedPartApp() {+ const app = express()+ app.put(+ '/external/gfs/uploads/:id/parts/:part',+ requireInternalToken,+ requireInternalService('external-rest-api'),+ (req, _res, next) => {+ ;(req as typeof req & { externalAuth: { userId: string } }).externalAuth = {+ userId: 'user-1',+ }+ next()+ },+ gfsUploadAdmission,+ (req, res) => {+ req.resume()+ req.once('end', () => res.status(204).end())+ }+ )+ return app+}+beforeEach(() => {acquireMock.mockReset()checkMock.mockReset()+ metricActiveInc.mockReset()+ metricBytesInc.mockReset()metricDec.mockReset()- metricInc.mockReset()+ metricRequestsInc.mockReset()releaseMock.mockReset()checkMock.mockResolvedValue(rateCheck())acquireMock.mockResolvedValue({@@ -128,6 +170,21 @@ describe('GFS upload admission', () => {expect(acquireMock).not.toHaveBeenCalled()})+ it('records a rejected part-body request without charging the byte counter', async () => {+ const response = await request(buildPartApp())+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')+ .set('content-type', 'application/offset+octet-stream')+ .set('content-length', '4')+ .set('upload-chunk-length', '3')+ .send(Buffer.from('abc'))++ expect(response.status).toBe(400)+ expect(metricRequestsInc).toHaveBeenCalledWith({ limit: 'part_body', result: 'rejected' }, 1)+ expect(metricRequestsInc).toHaveBeenCalledTimes(1)+ expect(metricBytesInc).not.toHaveBeenCalled()+ expect(metricActiveInc).not.toHaveBeenCalled()+ })+it('charges part bytes in fixed 1 MiB units before forwarding and releases its active slot', async () => {const bytes = Buffer.alloc(1024 * 1024 + 1, 0x61)const response = await request(buildPartApp())@@ -154,6 +211,55 @@ describe('GFS upload admission', () => {expect(releaseMock).toHaveBeenCalledTimes(1)})+ it('ignores a spoofed source IP unless the authenticated external relay asserted it', async () => {+ const bytes = Buffer.from('a')+ await request(buildPartApp())+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')+ .set('content-type', 'application/offset+octet-stream')+ .set('upload-chunk-length', String(bytes.length))+ .set('x-gfs-upload-source-ip', '203.0.113.7')+ .send(bytes)++ expect(checkMock.mock.calls[1][0]).not.toBe('gfs-upload:req:ip:203.0.113.7')+ checkMock.mockClear()++ await request(buildPartApp('external-rest-api'))+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')+ .set('content-type', 'application/offset+octet-stream')+ .set('upload-chunk-length', String(bytes.length))+ .set('x-gfs-upload-source-ip', '203.0.113.7')+ .send(bytes)++ expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')+ })++ it('binds source-IP assertion to the real internal service token wiring', async () => {+ const app = buildAuthenticatedPartApp()+ const path = '/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0'+ const headers = {+ 'content-type': 'application/offset+octet-stream',+ 'upload-chunk-length': '1',+ 'x-gfs-upload-source-ip': '203.0.113.7',+ }+ await request(app)+ .put(path)+ .auth('dev-rpc-proxy-token', { type: 'bearer' })+ .set('x-service-token', 'rpc-proxy')+ .set(headers)+ .send(Buffer.from('a'))+ .expect(401)++ checkMock.mockClear()+ await request(app)+ .put(path)+ .auth('dev-external-rest-api-token', { type: 'bearer' })+ .set('x-service-token', 'external-rest-api')+ .set(headers)+ .send(Buffer.from('a'))+ .expect(204)+ expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')+ })+it('returns the same 429 contract when the active-request budget is exhausted', async () => {acquireMock.mockResolvedValueOnce({allowed: false,diff --git a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.tsnew file mode 100644index 000000000..5e793c6b1--- /dev/null+++ b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts@@ -0,0 +1,245 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { createHash, randomBytes, randomUUID } from 'node:crypto'+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { Pool } from 'pg'+import { DbAuditSink } from '../../gfs-controller/src/authz/permissionClient.js'+import type { PermissionClient } from '../../gfs-controller/src/authz/permissionClient.js'+import { PgBlobStagingStore } from '../../gfs-controller/src/db/blobStaging.js'+import { GfsWriteService, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'+import { BlobStore } from '../../gfs-controller/src/storage/blobStore.js'+import { createGfsUploadFinalizer } from '../../gfs-controller/src/upload/uploadFinalizer.js'+import type {+ UploadPartRow,+ UploadSessionRow,+} from '../../gfs-controller/src/upload/uploadSession.js'+import { initDb } from '../src/db.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip+const BODY = Buffer.alloc(1024 * 1024, 0x42)+const BODY_SHA256 = createHash('sha256').update(BODY).digest('hex')+const HOST_SUBJECT = 'host:1st:gfs-test/upload-finalizer'++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++function session(uploadId: string, drive: string, parentRid: string): UploadSessionRow {+ return {+ uploadId,+ idempotencyKey: randomUUID(),+ drive,+ ownerSubject: HOST_SUBJECT,+ primarySubject: HOST_SUBJECT,+ operation: 'create',+ requestFingerprint: 'real-finalizer-test',+ parentRid,+ resourceRid: null,+ resourceName: 'payload.bin',+ ifMatch: null,+ expectedBytes: BODY.length,+ partBytes: BODY.length,+ partCount: 1,+ wholeSha256: BODY_SHA256,+ committedBytes: BODY.length,+ contiguousBytes: BODY.length,+ committedPartCount: 1,+ activePartCount: 0,+ sessionEpoch: 0,+ state: 'finalizing',+ resultResourceId: null,+ resultVersion: null,+ resultSha256: null,+ failureCode: null,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+}++function part(uploadId: string, root: string): UploadPartRow {+ return {+ uploadId,+ partNumber: 0,+ offsetBytes: 0,+ lengthBytes: BODY.length,+ sha256: BODY_SHA256,+ state: 'committed',+ stagingPath: join(root, '.uploads', uploadId, 'parts', '0.part'),+ leaseEpoch: 0,+ }+}++function permissions(allowed: () => boolean): PermissionClient {+ return {+ permissionEpoch: () => ({ generation: 0, bypassed: false }),+ authorize: async () => ({ allowed: allowed() }),+ } as unknown as PermissionClient+}++describeRealPostgres('GFS upload finalizer on real PostgreSQL + BlobStore', () => {+ const database = `gfs_upload_finalizer_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ const drive = `finalizer-${randomBytes(4).toString('hex')}`+ let adminPool: Pool+ let pool: Pool+ let storageRoot: string+ let blobRoot: string+ let parentId: string++ beforeAll(async () => {+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ pool = new Pool({ connectionString })+ await initDb({ connect: () => pool.connect() })+ storageRoot = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-parts-'))+ blobRoot = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-blobs-'))+ parentId = randomUUID()+ await pool.query(+ `INSERT INTO gfs_resources+ (resource_id, drive, parent_resource_id, name, kind, path_cache, version, bytes)+ VALUES ($1, $2, NULL, '', 'directory', '/', 0, 0)`,+ [parentId, drive]+ )+ }, 60_000)++ afterAll(async () => {+ await pool?.query('DELETE FROM gfs_resources WHERE drive = $1', [drive]).catch(() => undefined)+ await pool+ ?.query('DELETE FROM gfs_upload_sessions WHERE drive = $1', [drive])+ .catch(() => undefined)+ await pool?.end()+ await rm(storageRoot, { recursive: true, force: true }).catch(() => undefined)+ await rm(blobRoot, { recursive: true, force: true }).catch(() => undefined)+ if (!adminPool) return+ await adminPool.query(+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity+ WHERE datname = $1 AND pid <> pg_backend_pid()`,+ [database]+ )+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ await adminPool.end()+ })++ async function seedUpload(): Promise<{ upload: UploadSessionRow; uploadPart: UploadPartRow }> {+ const uploadId = randomUUID()+ const upload = session(uploadId, drive, parentId)+ const uploadPart = part(uploadId, storageRoot)+ await mkdir(join(storageRoot, '.uploads', uploadId, 'parts'), { recursive: true })+ await writeFile(uploadPart.stagingPath, BODY)+ await pool.query(+ `INSERT INTO gfs_upload_sessions+ (upload_id, idempotency_key, drive, owner_subject, primary_subject, operation,+ request_fingerprint, parent_rid, resource_name, expected_bytes, part_bytes,+ part_count, whole_sha256, committed_bytes, contiguous_bytes,+ committed_part_count, active_part_count, session_epoch, state, expires_at)+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)`,+ [+ upload.uploadId,+ upload.idempotencyKey,+ upload.drive,+ upload.ownerSubject,+ upload.primarySubject,+ upload.operation,+ upload.requestFingerprint,+ upload.parentRid,+ upload.resourceName,+ upload.expectedBytes,+ upload.partBytes,+ upload.partCount,+ upload.wholeSha256,+ upload.committedBytes,+ upload.contiguousBytes,+ upload.committedPartCount,+ upload.activePartCount,+ upload.sessionEpoch,+ upload.state,+ upload.expiresAt,+ ]+ )+ await pool.query(+ `INSERT INTO gfs_upload_parts+ (upload_id, part_number, offset_bytes, length_bytes, sha256, state,+ staging_path, lease_epoch, committed_at)+ VALUES ($1, 0, 0, $2, $3, 'committed', $4, 0, now())`,+ [upload.uploadId, BODY.length, BODY_SHA256, uploadPart.stagingPath]+ )+ return { upload, uploadPart }+ }++ function finalizer(allow: () => boolean): ReturnType<typeof createGfsUploadFinalizer> {+ const blobs = new BlobStore(blobRoot, 'writer')+ const writes = new GfsWriteService(new PgTransactor(pool), blobs, new PgBlobStagingStore(pool))+ return createGfsUploadFinalizer({+ pool,+ storageMountPath: storageRoot,+ permissions: permissions(allow),+ writeService: writes,+ audit: new DbAuditSink(pool),+ })+ }++ it('publishes the resource and receipt atomically through the real writer', async () => {+ const { upload, uploadPart } = await seedUpload()+ const result = await finalizer(() => true)(upload, [uploadPart])+ expect(result.sha256).toBe(BODY_SHA256)+ const receipt = await pool.query(+ `SELECT state, result_resource_id, result_version, result_sha256+ FROM gfs_upload_sessions WHERE upload_id = $1`,+ [upload.uploadId]+ )+ expect(receipt.rows[0]).toMatchObject({+ state: 'completed',+ result_resource_id: result.resourceId,+ result_version: result.version,+ result_sha256: BODY_SHA256,+ })+ const resource = await pool.query(+ `SELECT drive, parent_resource_id, bytes, content_sha256+ FROM gfs_resources WHERE resource_id = $1`,+ [result.resourceId]+ )+ expect(resource.rows[0]).toMatchObject({+ drive,+ parent_resource_id: parentId,+ bytes: BODY.length,+ content_sha256: BODY_SHA256,+ })+ const manifests = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,+ [result.resourceId]+ )+ expect(manifests.rows[0]?.count).toBe(0)+ }, 30_000)++ it('rolls back the real resource and receipt when authorization is denied', async () => {+ const { upload, uploadPart } = await seedUpload()+ await expect(finalizer(() => false)(upload, [uploadPart])).rejects.toMatchObject({+ code: 'forbidden',+ })+ const receipt = await pool.query(+ `SELECT state, result_resource_id FROM gfs_upload_sessions WHERE upload_id = $1`,+ [upload.uploadId]+ )+ expect(receipt.rows[0]).toEqual({ state: 'finalizing', result_resource_id: null })+ const resource = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_resources WHERE resource_id = $1`,+ [upload.uploadId]+ )+ expect(resource.rows[0]?.count).toBe(0)+ const manifests = await pool.query(+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,+ [upload.uploadId]+ )+ expect(manifests.rows[0]?.count).toBe(0)+ }, 30_000)+})diff --git a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts b/control-api/test/services.rateLimiter.realPostgres.integration.test.tsindex 2879879ce..4d086c91f 100644--- a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts@@ -2,7 +2,10 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'import { randomBytes } from 'node:crypto'import { Pool } from 'pg'import { initDb } from '../src/db.js'-import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'+import {+ acquireRateLimitConcurrencyLease,+ checkAndIncrementWithQuery,+} from '../src/services/rateLimiterService.js'const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URLconst describeRealPostgres = adminUrl ? describe : describe.skip@@ -69,29 +72,82 @@ describeRealPostgres('rate limiter atomicity on real PostgreSQL', () => {expect(Number(persisted.rows[0]?.count)).toBe(32)})- it('serializes a bounded advisory slot across independent PostgreSQL sessions', async () => {+ it('serializes the production bounded advisory lease across independent PostgreSQL sessions', async () => {const key = `gfs-upload-slot:${randomBytes(8).toString('hex')}`const first = await pool.connect()const second = await pool.connect()try {- const acquiredFirst = await first.query<{ acquired: boolean }>(- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,- [key]+ const firstLease = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: key, maxConcurrent: 1 }],+ { client: first })- const acquiredSecond = await second.query<{ acquired: boolean }>(- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,- [key]+ expect(firstLease).toMatchObject({ allowed: true, backendAvailable: true })++ const secondLease = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: key, maxConcurrent: 1 }],+ { client: second }+ )+ expect(secondLease).toMatchObject({ allowed: false, backendAvailable: true })++ await firstLease.release()+ const acquiredAfterRelease = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: key, maxConcurrent: 1 }],+ { client: second }+ )+ expect(acquiredAfterRelease).toMatchObject({ allowed: true, backendAvailable: true })+ await acquiredAfterRelease.release()+ } finally {+ first.release()+ second.release()+ }+ })++ it('uses distinct slots for capacity two and rolls back partial multi-bucket acquisition', async () => {+ const slotKey = `gfs-upload-slots:${randomBytes(8).toString('hex')}`+ const rollbackKey = `gfs-upload-rollback:${randomBytes(8).toString('hex')}`+ const freeKey = `gfs-upload-free:${randomBytes(8).toString('hex')}`+ const first = await pool.connect()+ const second = await pool.connect()+ try {+ const firstSlot = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: slotKey, maxConcurrent: 2 }],+ { client: first }+ )+ const secondSlot = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: slotKey, maxConcurrent: 2 }],+ { client: second })- expect(acquiredFirst.rows[0]?.acquired).toBe(true)- expect(acquiredSecond.rows[0]?.acquired).toBe(false)+ expect(firstSlot.allowed).toBe(true)+ expect(secondSlot.allowed).toBe(true)+ const exhausted = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: slotKey, maxConcurrent: 2 }],+ { client: first }+ )+ expect(exhausted.allowed).toBe(false)- await first.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])- const acquiredAfterRelease = await second.query<{ acquired: boolean }>(- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,- [key]+ const held = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: rollbackKey, maxConcurrent: 1 }],+ { client: first }+ )+ expect(held.allowed).toBe(true)+ const partial = await acquireRateLimitConcurrencyLease(+ [+ { bucketKey: freeKey, maxConcurrent: 1 },+ { bucketKey: rollbackKey, maxConcurrent: 1 },+ ],+ { client: second })- expect(acquiredAfterRelease.rows[0]?.acquired).toBe(true)- await second.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])+ expect(partial.allowed).toBe(false)+ const afterRollback = await acquireRateLimitConcurrencyLease(+ [{ bucketKey: freeKey, maxConcurrent: 1 }],+ { client: second }+ )+ expect(afterRollback.allowed).toBe(true)++ await firstSlot.release()+ await secondSlot.release()+ await held.release()+ await afterRollback.release()} finally {first.release()second.release()diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.tsindex a3152aeeb..312782d4a 100644--- a/control-api/test/services.rateLimiter.test.ts+++ b/control-api/test/services.rateLimiter.test.ts@@ -156,6 +156,26 @@ describe('rateLimiterService', () => {expect(mockPoolConnect).toHaveBeenCalledTimes(1)})+ it('destroys the dedicated connection when an advisory unlock outcome is unknown', async () => {+ const lease = await acquireRateLimitConcurrencyLease([+ { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },+ ])+ expect(lease.allowed).toBe(true)+ mockClientQuery.mockRejectedValueOnce(new Error('connection lost during unlock'))++ await lease.release()++ expect(mockConcurrencyClient.release).toHaveBeenCalledWith(true)+ // The fake client does not emulate PostgreSQL's session teardown; model+ // the advisory lock release that the destroyed real connection provides.+ advisoryLocks.clear()+ const replacement = await acquireRateLimitConcurrencyLease([+ { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },+ ])+ expect(replacement).toMatchObject({ allowed: true, backendAvailable: true })+ await replacement.release()+ })+it('fails open when pool.query throws (DB error)', async () => {mockPoolQuery.mockRejectedValueOnce(new Error('connection refused'))const r = await checkAndIncrement('test:bucket:failopen', 5)diff --git a/control-ui/app/control-api/[...path]/route.ts b/control-ui/app/control-api/[...path]/route.tsindex 27bb44bf7..e8e6dba9d 100644--- a/control-ui/app/control-api/[...path]/route.ts+++ b/control-ui/app/control-api/[...path]/route.ts@@ -134,7 +134,9 @@ function buildUpstreamUrl(req: NextRequest, path: string[]): string {function isGfsUploadRoute(req: NextRequest, path: string[]): boolean {if (req.method !== 'PUT') return falseconst pathname = `/${path.join('/')}`- return /^\/api\/v1\/gfs\/proxy\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i.test(pathname)+ return /^\/api\/v1\/gfs\/proxy\/v1\/uploads\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/parts\/[0-9]+$/i.test(+ pathname+ )}function copyRequestHeaders(req: NextRequest): Headers {diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsxindex 57d3f5e14..7b6174fb7 100644--- a/control-ui/components/__tests__/GfsBrowser.test.tsx+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx@@ -384,6 +384,10 @@ describe('GfsBrowser', () => {// Simulate an out-of-order in-flight part reporting a later absolute// offset before the accumulator commits the lower-numbered part.uploadInput?.onProgress?.({ uploadedBytes: 3 })+ // A failed retry clears that part's in-flight contribution. The UI+ // intentionally permits the truthful downward correction; the+ // monotonic guard applies only to stale uploading snapshots.+ uploadInput?.onProgress?.({ uploadedBytes: 1 })uploadInput?.onState?.({state: 'uploading',session: { ...completed, state: 'uploading' },@@ -441,10 +445,10 @@ describe('GfsBrowser', () => {const progress = await within(uploadDialog).findByRole('progressbar', {name: /Upload progress/,})- expect(progress).toHaveAttribute('value', '3')+ expect(progress).toHaveAttribute('value', '2')expect(progress).toHaveAttribute('aria-valuemin', '0')expect(progress).toHaveAttribute('aria-valuemax', '4')- expect(progress).toHaveAttribute('aria-valuenow', '3')+ expect(progress).toHaveAttribute('aria-valuenow', '2')fireEvent.click(within(uploadDialog).getByRole('button', { name: 'Pause' }))await waitFor(() => expect(job.pause).toHaveBeenCalledTimes(1))expect(await within(uploadDialog).findByRole('button', { name: 'Resume' })).toBeTruthy()diff --git a/control-ui/e2e/gfs-upload-v2.spec.ts b/control-ui/e2e/gfs-upload-v2.spec.tsindex be0c0d86b..ac665c15d 100644--- a/control-ui/e2e/gfs-upload-v2.spec.ts+++ b/control-ui/e2e/gfs-upload-v2.spec.ts@@ -11,8 +11,14 @@ import {import {GFS_UPLOAD_V2_BOUNDARIES,createDiskUploadFixture,+ createOversizedDiskUploadFixture,removeDiskUploadFixture,} from '../../tests/e2e/gfsUploadV2Fixtures'+import {+ readGfsUploadV2Enabled,+ restartGfsWriter,+ setGfsUploadV2Enabled,+} from '../../tests/e2e/gfsUploadV2Runtime'import { loginControlUi, openGlobalFileSystemFromSidebar } from './support/gfs-control-ui-session'test.describe.configure({ mode: 'serial' })@@ -211,3 +217,155 @@ test.describe('GFS Upload v2 — Control UI large-upload project', () => {})}})++test.describe('GFS Upload v2 — approved negative Control UI journeys', () => {+ test.skip(+ process.env.GFS_UPLOAD_V2_NEGATIVE_E2E !== '1',+ 'Set GFS_UPLOAD_V2_NEGATIVE_E2E=1 only on an owned non-production dev host with the profile-scoped random URLs.'+ )+ test.setTimeout(45 * 60_000)++ test('rejects a 209715201-byte file through the visible upload modal', async ({ page }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-oversize-ui')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ const source = await createOversizedDiskUploadFixture('.parquet', fixtureName)+ try {+ const current = await openWritableFolder(page, fixture.name)+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()+ const dialog = page.getByRole('dialog', { name: 'Upload file' })+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()+ await expect(+ page.getByRole('status').filter({ hasText: 'GFS uploads are limited to 200 MB per file.' })+ ).toBeVisible({ timeout: 15_000 })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 15_000 }+ )+ .toBeNull()+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toHaveCount(+ 0+ )+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })++ test('falls back to the legacy path when v2 is disabled', async ({ page }) => {+ const previous = await readGfsUploadV2Enabled()+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-legacy-ui')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ const source = await createDiskUploadFixture(2 * 1024 * 1024, '.bin', fixtureName)+ try {+ await setGfsUploadV2Enabled(false)+ const current = await openWritableFolder(page, fixture.name)+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()+ const dialog = page.getByRole('dialog', { name: 'Upload file' })+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()+ await expect(+ page+ .getByRole('status')+ .filter({ hasText: 'Resumable upload is unavailable; using the legacy 16 MiB path.' })+ ).toBeVisible({ timeout: 30_000 })+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({+ timeout: 120_000,+ })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toBeVisible()+ } finally {+ try {+ await setGfsUploadV2Enabled(previous)+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ }+ })++ test('recovers a visible upload after the writer deployment restarts', async ({ page }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-restart-ui')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)+ try {+ const current = await openWritableFolder(page, fixture.name)+ const capabilitiesResponse = page.waitForResponse(+ response =>+ response.request().method() === 'GET' &&+ response.url().includes('/gfs/proxy/v1/capabilities')+ )+ const createResponse = page.waitForResponse(+ response =>+ response.request().method() === 'POST' &&+ response.url().includes('/gfs/proxy/v1/uploads') &&+ !response.url().endsWith('/complete')+ )+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()+ const dialog = page.getByRole('dialog', { name: 'Upload file' })+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()+ const capabilities = await capabilitiesResponse+ const capabilitiesBody = (await capabilities.json()) as {+ upload?: { resumableV2?: { enabled?: boolean } }+ data?: { upload?: { resumableV2?: { enabled?: boolean } } }+ }+ const resumable =+ capabilitiesBody.upload?.resumableV2 ?? capabilitiesBody.data?.upload?.resumableV2+ expect(+ capabilities.status(),+ `${capabilities.url()} ${JSON.stringify(capabilitiesBody)}`+ ).toBe(200)+ expect(+ resumable?.enabled,+ 'restart journey must exercise the v2 writer, not legacy fallback'+ ).toBe(true)+ const created = await createResponse+ const createdBody = await created.text()+ expect(created.status(), `${created.url()} ${createdBody}`).toBeGreaterThanOrEqual(200)+ expect(createdBody).toMatch(/uploadId|upload_id|data/)+ const progress = page.getByRole('progressbar', {+ name: `Upload progress for ${source.fileName}`,+ })+ // Establish the visible in-flight state, then inject a real writer pod+ // loss. Completion after this point is the recovery assertion; the+ // positive journeys separately enforce two intermediate progress values.+ await expect(progress).toBeVisible({ timeout: 60_000 })+ await restartGfsWriter()+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({+ timeout: 1_200_000,+ })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toBeVisible()+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })+})diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.tsindex 11ce87cea..17f86ec52 100644--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts@@ -495,6 +495,108 @@ describe('GfsUploadJob', () => {expect(FakeXhr.requests).toHaveLength(2)})+ it('uses the extended bounded part budget while a writer is restarting', async () => {+ const file = new File([new Uint8Array([23])], 'part-writer-restart.bin')+ const session = {+ uploadId: '14141414-1414-4141-8141-141414141414',+ drive: 'main',+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ }+ FakeXhr.statuses = [502, 502, 502, 502, 502, 204]+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/uploads'))+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })+ if (method === 'HEAD') return new Response(null, { status: 204 })+ if (method === 'GET' && url.includes('/status'))+ return new Response(JSON.stringify({ ok: true, data: { session, parts: [] } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/complete'))+ return new Response(+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),+ { status: 200 }+ )+ throw new Error(`unexpected ${method} ${url}`)+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-part-writer-restart' },+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(FakeXhr.requests).toHaveLength(6)+ })++ it('removes a failed part attempt from aggregate progress before retrying it', async () => {+ const file = new File([new Uint8Array([31, 32, 33, 34])], 'progress-retry.bin')+ const session = {+ uploadId: '16161616-1616-4161-8161-161616161616',+ drive: 'main',+ expectedBytes: 4,+ partBytes: 4,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ }+ FakeXhr.statuses = [502, 204]+ const progress: number[] = []+ vi.stubGlobal(+ 'fetch',+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {+ const method = init?.method ?? 'GET'+ const url = String(_input)+ if (method === 'GET' && url.endsWith('/capabilities'))+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/uploads'))+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })+ if (method === 'HEAD') return new Response(null, { status: 204 })+ if (method === 'GET' && url.includes('/status'))+ return new Response(JSON.stringify({ ok: true, data: { session, parts: [] } }), {+ status: 200,+ })+ if (method === 'POST' && url.endsWith('/complete'))+ return new Response(+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),+ { status: 200 }+ )+ throw new Error(`unexpected ${method} ${url}`)+ })+ )++ await expect(+ new GfsUploadJob({+ file,+ name: file.name,+ target: { operation: 'create', parentRid: 'parent-progress-retry' },+ onProgress: value => progress.push(value.uploadedBytes),+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })++ expect(progress.some((value, index) => index > 0 && value < progress[index - 1]!)).toBe(true)+ })+it('adopts a status-confirmed committed part after its XHR response is lost', async () => {const file = new File([new Uint8Array([5, 6, 7, 8])], 'response-loss.bin', {lastModified: 13,diff --git a/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.ts b/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.tsnew file mode 100644index 000000000..05783ff10--- /dev/null+++ b/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from 'vitest'+import { validateGfsRuntimeImageMarker } from '../../../tests/e2e/gfsUploadV2Runtime'++const base = {+ clusterFingerprint: 'cluster-a',+ imagesGeneratedAt: '2026-08-12T18:00:00Z',+}++describe('GFS Upload v2 runtime image marker contract', () => {+ it('accepts local image mode without a registry tag', () => {+ expect(() =>+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'local', imageTag: '' })+ ).not.toThrow()+ })++ it('requires an immutable tag for GHCR image mode', () => {+ expect(() =>+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'ghcr', imageTag: '' })+ ).toThrow(/GHCR image marker/)+ })++ it('rejects a marker that omits the imageTag field entirely', () => {+ expect(() => validateGfsRuntimeImageMarker({ ...base, imageSource: 'local' })).toThrow(+ /lacks cluster fingerprint/+ )+ })++ it('rejects an image source outside the supported acquisition modes', () => {+ expect(() =>+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'unknown', imageTag: '' })+ ).toThrow(/unsupported imageSource/)+ })++ it('rejects a marker whose fingerprint or image acquisition stamp is stale', () => {+ expect(() =>+ validateGfsRuntimeImageMarker(+ { ...base, imageSource: 'local', imageTag: '' },+ { imageSource: 'local', imageTag: '', clusterFingerprint: 'cluster-b' }+ )+ ).toThrow(/clusterFingerprint/)+ expect(() =>+ validateGfsRuntimeImageMarker(+ { ...base, imageSource: 'local', imageTag: '' },+ { imagesGeneratedAt: '2026-08-12T19:00:00Z' }+ )+ ).toThrow(/imagesGeneratedAt/)+ })+})diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.tsindex 9fd740f6e..536b54d86 100644--- a/control-ui/lib/gfsFileUpload.ts+++ b/control-ui/lib/gfsFileUpload.ts@@ -14,6 +14,12 @@ const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3+/**+ * A writer restart can keep the service endpoint unavailable for longer than+ * the ordinary application-error retry budget. Keep this separate from the+ * lifecycle budget so 502/503/504 recovery is longer but still bounded.+ */+export const GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS = 6export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000@@ -517,6 +523,13 @@ function ambiguousPartResponse(status: number | undefined): boolean {return status === undefined || isAmbiguousUploadStatus(status)}+function partRetryAttempts(error: unknown): number {+ const status = statusOf(error)+ return status === undefined || status === 502 || status === 503 || status === 504+ ? GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS+ : GFS_UPLOAD_RETRY_MAX_ATTEMPTS+}+function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {return new Error(`upload_part_outcome_unknown: could not prove whether part ${partNumber} committed; resume the persisted session`,@@ -662,7 +675,8 @@ async function uploadPartWithRetries(const blob = file.slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)const checksum = await sha256Part(blob)let lastError: unknown- for (let attempt = 0; attempt < 3; attempt += 1) {+ let maxAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS+ for (let attempt = 0; attempt < maxAttempts; attempt += 1) {try {await putPart(session.uploadId,@@ -680,6 +694,7 @@ async function uploadPartWithRetries(lastError = errorif (signal?.aborted) throw signal.reason ?? errorif (!assertRetryable(error)) throw error+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(error))onRetryableFailure?.()if (ambiguousPartResponse(statusOf(error))) {const reconciled = await reconcileAmbiguousPart(file, session, part, checksum, signal)@@ -688,7 +703,7 @@ async function uploadPartWithRetries(return}}- if (attempt === 2) throw error+ if (attempt + 1 >= maxAttempts) throw errorawait delay(retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),signal@@ -707,6 +722,7 @@ async function runParts(shouldPause: () => boolean,onProgress?: (progress: GfsUploadProgress) => void,onByteProgress?: (progress: GfsUploadProgress) => void,+ onRetryableFailure?: (partNumber: number) => void,onPartFailure?: (partNumber: number) => void,fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY,instabilityFailureThreshold = GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD@@ -759,7 +775,10 @@ async function runParts(parts.length,signal,onByteProgress,- registerRetryableFailure,+ () => {+ registerRetryableFailure()+ onRetryableFailure?.(part.partNumber)+ },registerPartSuccess)onProgress?.({@@ -1267,6 +1286,17 @@ export class GfsUploadJob {() => this.pauseRequested || this.canceled,progress,byteProgress,+ partNumber => {+ inFlightLoaded.delete(partNumber)+ this.uploadedBytes = calculateUploadedBytes()+ this.input.onProgress?.({+ uploadedBytes: this.uploadedBytes,+ totalBytes: this.input.file.size,+ partNumber,+ partCount: allParts.length,+ })+ this.emit()+ },partNumber => {inFlightLoaded.delete(partNumber)this.uploadedBytes = calculateUploadedBytes()diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.tsindex fa12dbd6b..579816668 100644--- a/desktop-app/src/__tests__/appService.gfs.test.ts+++ b/desktop-app/src/__tests__/appService.gfs.test.ts@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'import { join } from 'node:path'import { AppService, legacyEncodedFile, migrateDesktopGfsUploadState } from '../appService.js'import { config, getActiveEnvKey } from '../config.js'+import { DesktopUploadCapabilityError } from '../gfs/upload.js'vi.mock('../chatStoreBinding.js', () => ({bindChatStoreForUser: vi.fn(),@@ -212,6 +213,10 @@ type UploadScopeTestService = {scope: ReturnType<typeof scopedUploadRecord>['scope']}>+ gfsClient: {+ createResource: ReturnType<typeof vi.fn>+ replaceFile: ReturnType<typeof vi.fn>+ }desktopGfsUploadStatePath: () => Promise<string>startDesktopGfsUpload: ReturnType<typeof vi.fn>authClient: {@@ -225,12 +230,16 @@ type UploadScopeTestService = {completePasswordLogin: (email: string, password: string) => ReturnType<AppService['googleLogin']>googleLogin: AppService['googleLogin']listGfsUploadSessions: AppService['listGfsUploadSessions']+ getGfsUploadSnapshot: AppService['getGfsUploadSnapshot']+ pauseGfsUpload: AppService['pauseGfsUpload']+ cancelGfsUpload: AppService['cancelGfsUpload']resumeGfsUpload: AppService['resumeGfsUpload']logout: AppService['logout']runScopedLegacyGfsUpload: <T>(scope: ReturnType<typeof scopedUploadRecord>['scope'],operation: (token: string, signal: AbortSignal) => Promise<T>) => Promise<T>+ startGfsFileUpload: AppService['startGfsFileUpload']}function authenticatedUploadService(statePath: string): UploadScopeTestService {@@ -267,6 +276,116 @@ function authenticatedUploadService(statePath: string): UploadScopeTestService {}describe('AppService GFS upload security scope', () => {+ it('falls back to the bounded legacy create path for a fresh start when v2 is disabled', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-legacy-start-fallback-'))+ try {+ const filePath = join(root, 'legacy.bin')+ await writeFile(filePath, Buffer.from('legacy payload'))+ const service = authenticatedUploadService(join(root, 'gfs-upload-sessions.json'))+ service.startDesktopGfsUpload.mockRejectedValue(+ new DesktopUploadCapabilityError('resumable uploads are disabled')+ )+ service.gfsClient = {+ createResource: vi.fn().mockResolvedValue({+ resourceId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',+ rid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ gfsUri: 'gfs://main/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ drive: 'main',+ parentResourceId: 'parent-rid',+ name: 'legacy.bin',+ kind: 'file',+ path: '/legacy.bin',+ version: 1,+ bytes: 14,+ }),+ replaceFile: vi.fn(),+ }++ const result = await service.startGfsFileUpload('parent-rid', 'legacy.bin', filePath, 'main')++ expect(result).toMatchObject({+ state: 'completed',+ operation: 'create',+ resultResourceId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',+ expectedBytes: 14,+ })+ expect(service.gfsClient.createResource).toHaveBeenCalledWith(+ expect.objectContaining({+ parentResourceId: 'parent-rid',+ name: 'legacy.bin',+ drive: 'main',+ kind: 'file',+ encodedData: Buffer.from('legacy payload').toString('base64'),+ }),+ 'token-a',+ expect.any(AbortSignal)+ )+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })++ it('keeps existing upload controls and read state available during a transient team hop', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-transient-controls-'))+ try {+ const statePath = join(root, 'gfs-upload-sessions.json')+ const record = scopedUploadRecord('92929292-9292-4292-8292-929292929292')+ await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))+ const service = authenticatedUploadService(statePath) as UploadScopeTestService & {+ gfsTransientTeamHopDepth: number+ }+ const job = {+ snapshot: vi.fn(() => ({+ state: 'uploading',+ session: record.session,+ uploadedBytes: 0,+ totalBytes: record.fileSize,+ })),+ pause: vi.fn(async () => ({ ...record.session, state: 'paused' as const })),+ cancel: vi.fn(async () => undefined),+ }+ service.gfsTransientTeamHopDepth = 1+ service.gfsUploadJobs.set(record.uploadId, {+ job,+ promise: Promise.resolve(record.session),+ scope: record.scope,+ })++ await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({+ state: 'uploading',+ })+ await service.pauseGfsUpload(record.uploadId, 'main')+ await service.cancelGfsUpload(record.uploadId, 'main')+ expect(job.pause).toHaveBeenCalledTimes(1)+ expect(job.cancel).toHaveBeenCalledTimes(1)+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })++ it('maps a persisted active record to the renderer uploading state after restart', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-snapshot-state-'))+ try {+ const statePath = join(root, 'gfs-upload-sessions.json')+ const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')+ const service = authenticatedUploadService(statePath)+ // The normal state loader converts orphaned active records to+ // suspended_auth. This injects the already-migrated persisted branch so+ // the IPC boundary itself is still proven not to leak `active`.+ ;(service as any).readDesktopGfsUploadState = vi+ .fn()+ .mockResolvedValue({ version: 2, records: [record], quarantined: [] })++ await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({+ state: 'uploading',+ uploadedBytes: 0,+ totalBytes: 4,+ })+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })+it('quarantines legacy unscoped state instead of restoring it', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-state-migration-'))try {diff --git a/desktop-app/src/__tests__/appService.rpc.test.ts b/desktop-app/src/__tests__/appService.rpc.test.tsindex 26de93281..3242edb1f 100644--- a/desktop-app/src/__tests__/appService.rpc.test.ts+++ b/desktop-app/src/__tests__/appService.rpc.test.ts@@ -250,6 +250,110 @@ describe('AppService.invokeHostMessage', () => {expect(service.gfsTransientTeamHopDepth).toBe(0)})+ it('keeps existing GFS controls usable through the real team-hop lifecycle', async () => {+ const service = new AppService() as any+ const statePath = path.join(chatStoreBaseDir, 'gfs-upload-state.json')+ const uploadId = '92929292-9292-4292-8292-929292929292'+ const session = {+ uploadId,+ drive: 'main',+ operation: 'create',+ expectedBytes: 4,+ partBytes: 4,+ partCount: 1,+ state: 'uploading',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+ const scope = {+ ownerId: 'user-1',+ teamId: 'team-a',+ environmentKey: 'test-env',+ baseUrl: 'https://external.example',+ drive: 'main',+ authEpoch: 1,+ }+ await fs.writeFile(+ statePath,+ JSON.stringify({+ version: 2,+ records: [+ {+ version: 2,+ uploadId,+ filePath: '/tmp/payload.bin',+ fileName: 'payload.bin',+ fileSize: 4,+ target: { operation: 'create', parentRid: 'parent' },+ name: 'payload.bin',+ session,+ scope,+ status: 'active',+ updatedAt: new Date().toISOString(),+ },+ ],+ quarantined: [],+ })+ )+ service.desktopGfsUploadStatePath = vi.fn().mockResolvedValue(statePath)+ service.sessionToken = 'team-a-token'+ service.me = {+ id: 'user-1',+ email: '[email protected]',+ name: 'Test User',+ picture: null,+ teamId: 'team-a',+ teamName: 'Team A',+ role: 'member',+ }+ service.gfsAuthEpoch = 1+ service.gfsDispatchBlocked = false+ service.gfsScopeIdentity = { ...scope, authEpoch: undefined }+ const job = {+ snapshot: vi.fn(() => ({ state: 'uploading', session, uploadedBytes: 0, totalBytes: 4 })),+ pause: vi.fn(async () => ({ ...session, state: 'paused' })),+ resume: vi.fn(async () => ({ ...session, state: 'uploading' })),+ cancel: vi.fn(async () => undefined),+ }+ service.gfsUploadJobs = new Map([[uploadId, { job, promise: Promise.resolve(session), scope }]])+ service.authClient = {+ getMe: vi+ .fn()+ .mockResolvedValueOnce({ ...service.me, teamId: 'team-b', teamName: 'Team B' })+ .mockResolvedValueOnce(service.me),+ switchTeam: vi.fn(async (_token: string, teamId: string) => ({+ token: `${teamId}-token`,+ team: { id: teamId, name: teamId, role: 'member' },+ })),+ }+ service.tokenStore = { setSessionToken: vi.fn() }++ await service.runWithTeamContext('team-b', async () => {+ await expect(service.getGfsUploadSnapshot(uploadId, 'main')).resolves.toMatchObject({+ state: 'uploading',+ })+ await expect(service.listGfsUploadSessions('main')).resolves.toHaveLength(1)+ await expect(service.pauseGfsUpload(uploadId, 'main')).resolves.toMatchObject({+ state: 'paused',+ })+ await expect(service.resumeGfsUpload(uploadId, 'main')).resolves.toMatchObject({+ state: 'uploading',+ })+ await expect(service.cancelGfsUpload(uploadId, 'main')).resolves.toBeUndefined()+ })++ expect(job.snapshot).toHaveBeenCalledTimes(1)+ expect(job.pause).toHaveBeenCalledTimes(1)+ expect(job.resume).toHaveBeenCalledTimes(1)+ expect(job.cancel).toHaveBeenCalledTimes(1)+ expect(service.me.teamId).toBe('team-a')+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(1, 'team-a-token', 'team-b')+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(2, 'team-b-token', 'team-a')+ })+it('fences the old GFS scope if a replacement team token cannot be refreshed', async () => {const service = new AppService() as anyservice.sessionToken = 'team-a-token'diff --git a/desktop-app/src/appService.ts b/desktop-app/src/appService.tsindex 405e2fee6..79ce379b7 100644--- a/desktop-app/src/appService.ts+++ b/desktop-app/src/appService.ts@@ -2136,14 +2136,33 @@ export class AppService {drive?: string,resumeUploadId?: string): Promise<DesktopUploadSession> {- return this.startDesktopGfsUpload({- filePath,- name,- drive: normalizeDesktopUploadDrive(drive),- operation: 'create',- parentRid: parentResourceId,- resumeUploadId,- })+ const canonicalDrive = normalizeDesktopUploadDrive(drive)+ const scope = this.currentDesktopGfsUploadScope(canonicalDrive)+ try {+ return await this.startDesktopGfsUpload({+ filePath,+ name,+ drive: canonicalDrive,+ operation: 'create',+ parentRid: parentResourceId,+ resumeUploadId,+ })+ } catch (error) {+ // A legacy fallback is valid only for a fresh upload. An explicit resume+ // must remain a v2 operation so a missing capability cannot silently turn+ // a persisted session into a second non-resumable resource.+ if (!(error instanceof DesktopUploadCapabilityError) || resumeUploadId) throw error+ const resource = await this.runScopedLegacyGfsUpload(scope, async (token, signal) => {+ const encodedData = await legacyEncodedFile(filePath)+ this.assertCurrentDesktopGfsUploadScope(scope)+ return this.gfsClient.createResource(+ { parentResourceId, drive: canonicalDrive, name, kind: 'file', encodedData },+ token,+ signal+ )+ })+ return legacyReceipt(resource, 'create')+ }}async startGfsFileReplace(@@ -2153,20 +2172,39 @@ export class AppService {ifMatch?: number,resumeUploadId?: string): Promise<DesktopUploadSession> {- return this.startDesktopGfsUpload({- filePath,- name: resourceId,- drive: normalizeDesktopUploadDrive(drive),- operation: 'replace',- resourceRid: resourceId,- ifMatch,- resumeUploadId,- })+ const canonicalDrive = normalizeDesktopUploadDrive(drive)+ const scope = this.currentDesktopGfsUploadScope(canonicalDrive)+ try {+ return await this.startDesktopGfsUpload({+ filePath,+ name: resourceId,+ drive: canonicalDrive,+ operation: 'replace',+ resourceRid: resourceId,+ ifMatch,+ resumeUploadId,+ })+ } catch (error) {+ if (!(error instanceof DesktopUploadCapabilityError) || resumeUploadId) throw error+ const resource = await this.runScopedLegacyGfsUpload(scope, async (token, signal) => {+ const encodedData = await legacyEncodedFile(filePath)+ this.assertCurrentDesktopGfsUploadScope(scope)+ return this.gfsClient.replaceFile(+ { resourceId, drive: canonicalDrive, ifMatch, encodedData },+ token,+ signal+ )+ })+ return legacyReceipt(resource, 'replace')+ }}async getGfsUploadSnapshot(uploadId: string, drive?: string) {- const scope = this.currentDesktopGfsUploadScope(drive)const entry = this.gfsUploadJobs.get(uploadId)+ // Snapshot is read-only and an existing job already owns its captured+ // token/scope. It must remain observable during a finite internal team hop;+ // only a brand-new user-facing dispatch is fenced by that hop.+ const scope = this.currentDesktopGfsUploadScope(drive, { allowTransientTeamHop: true })if (entry && sameDesktopGfsUploadScope(entry.scope, scope)) return entry.job.snapshot()const state = await this.readDesktopGfsUploadState()const record = state.records.find(@@ -2176,7 +2214,11 @@ export class AppService {)if (!record) return nullreturn {- state: record.status,+ // `active` is the persistence marker for a process-owned upload. The+ // renderer contract describes the same resumable state as `uploading`;+ // never leak the storage-only marker across the IPC boundary after a+ // restart.+ state: record.status === 'active' ? 'uploading' : record.status,session: record.session,uploadedBytes: record.session.committedBytes,totalBytes: record.fileSize,@@ -2184,8 +2226,10 @@ export class AppService {}async pauseGfsUpload(uploadId: string, drive?: string): Promise<DesktopUploadSession> {- const scope = this.currentDesktopGfsUploadScope(drive)const entry = this.gfsUploadJobs.get(uploadId)+ const scope = this.currentDesktopGfsUploadScope(drive, {+ allowTransientTeamHop: Boolean(entry),+ })if (!entry || !sameDesktopGfsUploadScope(entry.scope, scope))throw new Error('GFS upload is not active in the current security scope')const session = await entry.job.pause()@@ -2201,8 +2245,10 @@ export class AppService {}async resumeGfsUpload(uploadId: string, drive?: string): Promise<DesktopUploadSession> {- const scope = this.currentDesktopGfsUploadScope(drive)const entry = this.gfsUploadJobs.get(uploadId)+ const scope = this.currentDesktopGfsUploadScope(drive, {+ allowTransientTeamHop: Boolean(entry),+ })if (entry) {if (!sameDesktopGfsUploadScope(entry.scope, scope))throw new Error('GFS upload is not active in the current security scope')@@ -2231,8 +2277,10 @@ export class AppService {}async cancelGfsUpload(uploadId: string, drive?: string): Promise<void> {- const scope = this.currentDesktopGfsUploadScope(drive)const entry = this.gfsUploadJobs.get(uploadId)+ const scope = this.currentDesktopGfsUploadScope(drive, {+ allowTransientTeamHop: Boolean(entry),+ })if (entry) {if (!sameDesktopGfsUploadScope(entry.scope, scope))throw new Error('GFS upload is not active in the current security scope')@@ -2261,7 +2309,9 @@ export class AppService {}async listGfsUploadSessions(drive?: string): Promise<DesktopGfsUploadSummary[]> {- const scope = this.currentDesktopGfsUploadScope(drive)+ // Listing persisted state is read-only and must remain available while a+ // finite internal team hop temporarily borrows another team's token.+ const scope = this.currentDesktopGfsUploadScope(drive, { allowTransientTeamHop: true })const state = await this.readDesktopGfsUploadState()return state.records.filter(record => sameDesktopGfsUploadScope(record.scope, scope, { includeAuthEpoch: false }))diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.tsindex 341ba7d5c..7ddaee2f3 100644--- a/desktop-app/src/gfs/upload.test.ts+++ b/desktop-app/src/gfs/upload.test.ts@@ -239,6 +239,141 @@ describe('desktop GFS indexed uploader', () => {}})+ it('uses the extended bounded part budget while a writer is restarting', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-writer-restart-'))+ try {+ const filePath = join(root, 'payload.bin')+ await writeFile(filePath, Buffer.from([23]))+ const statuses = [502, 502, 502, 502, 502, 204]+ let partRequests = 0+ const session = {+ uploadId: '14141414-1414-4141-8141-141414141414',+ drive: 'main',+ operation: 'create' as const,+ expectedBytes: 1,+ partBytes: 1,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+ const transport = {+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))+ return enabledCapabilities() as T+ if (method === 'HEAD') return {} as T+ if (method === 'GET' && requestPath(url).endsWith('/status'))+ return { ok: true, data: { session, parts: [] } } as T+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))+ return { ok: true, data: session } as T+ if (method === 'POST' && requestPath(url).endsWith('/complete'))+ return { ok: true, data: { ...session, state: 'completed' } } as T+ throw new Error(`unexpected ${method} ${url}`)+ },+ async requestPart(+ _url: string,+ _token: string,+ _headers: Record<string, string>,+ body: NodeJS.ReadableStream+ ) {+ partRequests += 1+ for await (const _chunk of body as AsyncIterable<Buffer>) {+ /* drain */+ }+ return { status: statuses.shift() ?? 204, text: '' }+ },+ }++ await expect(+ new DesktopGfsUploadJob({+ baseUrl: 'https://api.example',+ token: 'token',+ filePath,+ name: 'payload.bin',+ drive: 'main',+ operation: 'create',+ parentRid: 'parent-writer-restart',+ transport,+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })+ expect(partRequests).toBe(6)+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })++ it('removes a failed part attempt from aggregate progress before retrying it', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-progress-retry-'))+ try {+ const filePath = join(root, 'payload.bin')+ await writeFile(filePath, Buffer.from([31, 32, 33, 34]))+ const session = {+ uploadId: '16161616-1616-4161-8161-161616161616',+ drive: 'main',+ operation: 'create' as const,+ expectedBytes: 4,+ partBytes: 4,+ partCount: 1,+ state: 'initiated',+ contiguousBytes: 0,+ committedBytes: 0,+ committedPartCount: 0,+ activePartCount: 0,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+ let partRequests = 0+ const progress: number[] = []+ const transport = {+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))+ return enabledCapabilities() as T+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))+ return { ok: true, data: session } as T+ if (method === 'GET' && requestPath(url).endsWith('/status'))+ return { ok: true, data: { session, parts: [] } } as T+ if (method === 'POST' && requestPath(url).endsWith('/complete'))+ return { ok: true, data: { ...session, state: 'completed' } } as T+ if (method === 'HEAD') return {} as T+ throw new Error(`unexpected ${method} ${url}`)+ },+ async requestPart(+ _url: string,+ _token: string,+ _headers: Record<string, string>,+ body: NodeJS.ReadableStream+ ) {+ for await (const _chunk of body as AsyncIterable<Buffer>) {+ /* drain */+ }+ partRequests += 1+ return { status: partRequests === 1 ? 502 : 204, text: '' }+ },+ }++ await expect(+ new DesktopGfsUploadJob({+ baseUrl: 'https://api.example',+ token: 'token',+ filePath,+ name: 'payload.bin',+ drive: 'main',+ operation: 'create',+ parentRid: 'parent-progress-retry',+ transport,+ onProgress: (uploadedBytes: number) => progress.push(uploadedBytes),+ }).start()+ ).resolves.toMatchObject({ state: 'completed' })++ expect(partRequests).toBe(2)+ expect(progress.some((value, index) => index > 0 && value < progress[index - 1]!)).toBe(true)+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })+it('resumes from sparse status and does not replay a committed part', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-resume-'))try {diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.tsindex 1a579e165..5e510b8ca 100644--- a/desktop-app/src/gfs/upload.ts+++ b/desktop-app/src/gfs/upload.ts@@ -14,6 +14,12 @@ const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3+/**+ * A writer restart can keep the service endpoint unavailable for longer than+ * the ordinary application-error retry budget. Keep this separate from the+ * lifecycle budget so 502/503/504 recovery is longer but still bounded.+ */+export const GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS = 6export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250@@ -263,6 +269,16 @@ function retryableError(error: unknown): boolean {return status === undefined || retryable(status)}+function partRetryAttempts(error: unknown): number {+ const status =+ error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'+ ? Number((error as { status: number }).status)+ : undefined+ return status === undefined || status === 502 || status === 503 || status === 504+ ? GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS+ : GFS_UPLOAD_RETRY_MAX_ATTEMPTS+}+/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */export function parseRetryAfter(value: string | null | undefined,@@ -685,6 +701,7 @@ async function putPart(onPartSuccess?: () => void): Promise<void> {let attempt = 0+ let maxAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTSfor (;;) {assertAuthEpoch(input)await assertSourceIdentity(input)@@ -732,6 +749,7 @@ async function putPart(} catch (error) {if (signal?.aborted) throw signal.reason ?? errorif (!retryableError(error)) throw error+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(error))onRetryableFailure?.()const reconciled = await reconcileAmbiguousPart(input,@@ -746,7 +764,7 @@ async function putPart(onPartSuccess?.()return}- if (attempt >= 2) throw error+ if (attempt + 1 >= maxAttempts) throw errorawait sleep(retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),signal@@ -761,6 +779,7 @@ async function putPart(if (!retryable(response.status)) {throw uploadPartError(response.status, response.text)}+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(response))onRetryableFailure?.()if (ambiguousPartResponse(response.status)) {const reconciled = await reconcileAmbiguousPart(@@ -777,7 +796,7 @@ async function putPart(return}}- if (attempt >= 2) throw uploadPartError(response.status, response.text)+ if (attempt + 1 >= maxAttempts) throw uploadPartError(response.status, response.text)await sleep(retryAfterOf(response) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),signal@@ -882,7 +901,11 @@ async function runParts()input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)},- registerRetryableFailure,+ () => {+ registerRetryableFailure()+ inFlightLoaded.delete(part.partNumber)+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)+ },registerPartSuccess)completedParts.add(part.partNumber)diff --git a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.tsindex 0e82e5f1f..09970a082 100644--- a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts+++ b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts@@ -14,8 +14,16 @@ import {import {GFS_UPLOAD_V2_BOUNDARIES,createDiskUploadFixture,+ createOversizedDiskUploadFixture,removeDiskUploadFixture,} from '../../../tests/e2e/gfsUploadV2Fixtures'+import {+ countGfsCreateSessions,+ readGfsUploadV2Enabled,+ restartGfsWriter,+ revokeGfsUserWriteGrant,+ setGfsUploadV2Enabled,+} from '../../../tests/e2e/gfsUploadV2Runtime'import { test } from './fixtures'import { openResourcesNavItem } from './navigationHelpers'@@ -413,3 +421,193 @@ test.describe('GFS Upload v2 — packaged Desktop project', () => {})}})++test.describe('GFS Upload v2 — approved negative Desktop journeys', () => {+ test.skip(+ process.env.GFS_UPLOAD_V2_NEGATIVE_E2E !== '1',+ 'Set GFS_UPLOAD_V2_NEGATIVE_E2E=1 only on an owned non-production dev host with profile-scoped random URLs.'+ )+ test.setTimeout(45 * 60_000)++ test('rejects a 209715201-byte file through the visible Desktop action', async ({ appPage }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-oversize-desktop')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ await seedGfsGrant({+ resourceId: fixture.resourceId,+ subjectType: 'user',+ subjectId: getE2EUserId(),+ permissions: ['read', 'write', 'delete'],+ inherit: true,+ grantedBy: 'e2e:gfs-upload-v2-negative',+ })+ const source = await createOversizedDiskUploadFixture('.parquet', fixtureName)+ try {+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)+ await selectFileThroughVisibleAction(+ appPage,+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),+ source.filePath+ )+ await expect(+ appPage+ .getByRole('alert')+ .filter({ hasText: 'GFS uploads are limited to 200 MB per file.' })+ ).toBeVisible({ timeout: 15_000 })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 15_000 }+ )+ .toBeNull()+ await expect(browser.getByRole('button', { name: source.fileName, exact: true })).toHaveCount(+ 0+ )+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })++ test('falls back to the legacy path when v2 is disabled', async ({ appPage }) => {+ const previous = await readGfsUploadV2Enabled()+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-legacy-desktop')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ await seedGfsGrant({+ resourceId: fixture.resourceId,+ subjectType: 'user',+ subjectId: getE2EUserId(),+ permissions: ['read', 'write', 'delete'],+ inherit: true,+ grantedBy: 'e2e:gfs-upload-v2-negative',+ })+ const source = await createDiskUploadFixture(2 * 1024 * 1024, '.bin', fixtureName)+ try {+ await setGfsUploadV2Enabled(false)+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)+ await selectFileThroughVisibleAction(+ appPage,+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),+ source.filePath+ )+ await expect(appPage.getByText(`Uploaded ${source.fileName}`, { exact: true })).toBeVisible({+ timeout: 120_000,+ })+ expect(countGfsCreateSessions(fixture.resourceId, source.fileName, getE2EUserId())).toBe(0)+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })+ await expect(+ browser.getByRole('button', { name: source.fileName, exact: true })+ ).toBeVisible()+ } finally {+ try {+ await setGfsUploadV2Enabled(previous)+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ }+ })++ test('recovers a visible upload after the writer deployment restarts', async ({ appPage }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-restart-desktop')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ await seedGfsGrant({+ resourceId: fixture.resourceId,+ subjectType: 'user',+ subjectId: getE2EUserId(),+ permissions: ['read', 'write', 'delete'],+ inherit: true,+ grantedBy: 'e2e:gfs-upload-v2-negative',+ })+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)+ try {+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)+ await selectFileThroughVisibleAction(+ appPage,+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),+ source.filePath+ )+ const progress = browser.getByRole('progressbar', {+ name: `GFS upload progress for ${source.fileName}`,+ })+ await expect(progress).toBeVisible({ timeout: 60_000 })+ await restartGfsWriter()+ await expect(appPage.getByText(`Uploaded ${source.fileName}`, { exact: true })).toBeVisible({+ timeout: 1_200_000,+ })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })++ test('fails visibly when the user grant is revoked during an upload', async ({ appPage }) => {+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-revoke-desktop')+ const fixture = seedGfsDirectoryFixture(fixtureName)+ const userId = getE2EUserId()+ seedGfsGrant({+ resourceId: fixture.resourceId,+ subjectType: 'user',+ subjectId: userId,+ permissions: ['read', 'write', 'delete'],+ inherit: true,+ grantedBy: 'e2e:gfs-upload-v2-negative',+ })+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)+ try {+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)+ await selectFileThroughVisibleAction(+ appPage,+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),+ source.filePath+ )+ const progress = browser.getByRole('progressbar', {+ name: `GFS upload progress for ${source.fileName}`,+ })+ await expect(progress).toBeVisible({ timeout: 60_000 })+ await revokeGfsUserWriteGrant(fixture.resourceId, userId)+ await expect(appPage.getByText('Upload needs attention', { exact: true })).toBeVisible({+ timeout: 120_000,+ })+ await expect+ .poll(+ () =>+ getGfsChildResourceSummary({+ parentResourceId: fixture.resourceId,+ name: source.fileName,+ }),+ { timeout: 60_000, intervals: [250, 1_000] }+ )+ .toBeNull()+ } finally {+ await removeDiskUploadFixture(source)+ cleanupGfsFixture(fixtureName)+ assertGfsFixtureCleaned(fixtureName)+ }+ })+})diff --git a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsxindex 227cae0c8..20b486a26 100644--- a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx+++ b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx@@ -128,6 +128,27 @@ function Probe() {)}+function UploadProbe() {+ const ctrl = useGfsBrowserController()+ return (+ <>+ <div data-testid="upload-state">{ctrl.uploadSnapshot?.state ?? 'none'}</div>+ <button+ type="button"+ onClick={() =>+ void ctrl.startFileUpload({+ parentResourceId: 'parent-rid',+ name: 'legacy.bin',+ filePath: '/tmp/legacy.bin',+ })+ }+ >+ start upload+ </button>+ </>+ )+}+function Harness({ children }: { children: ReactNode }) {const [me, setMe] = useState<AuthContextValue['me']>(userA)const [client] = useState(@@ -191,6 +212,51 @@ describe('useGfsBrowserController', () => {await waitFor(() => expect(screen.getByTestId('current').textContent).toBe('none'))})+ it('finishes a completed legacy receipt without polling it as a v2 session', async () => {+ const startFileUpload = vi.fn(async () => ({+ uploadId: 'legacy-resource-id',+ drive: 'main',+ operation: 'create' as const,+ expectedBytes: 12,+ partBytes: 12,+ partCount: 1,+ state: 'completed',+ contiguousBytes: 12,+ committedBytes: 12,+ committedPartCount: 1,+ activePartCount: 0,+ expiresAt: new Date().toISOString(),+ resultResourceId: 'legacy-resource-id',+ resultVersion: 1,+ }))+ const getUploadSnapshot = vi.fn()+ Object.defineProperty(window, 'clerum', {+ configurable: true,+ value: {+ gfs: {+ listAccessible: vi.fn(async () => ({ items: [], nextCursor: null })),+ startFileUpload,+ getUploadSnapshot,+ },+ },+ })++ render(<UploadProbe />, { wrapper: Harness })+ await act(async () => {+ screen.getByRole('button', { name: 'start upload' }).click()+ })++ await waitFor(() => expect(screen.getByTestId('upload-state').textContent).toBe('completed'))+ expect(startFileUpload).toHaveBeenCalledWith(+ 'parent-rid',+ 'legacy.bin',+ '/tmp/legacy.bin',+ 'main',+ undefined+ )+ expect(getUploadSnapshot).not.toHaveBeenCalled()+ })+it('refreshes cached affordances after permissions change outside Desktop', async () => {let held = ['read']const affordances = vi.fn(async () => ({diff --git a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.tsindex 4e82d3602..3fb242271 100644--- a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts+++ b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts@@ -300,12 +300,20 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {input.resumeUploadId)uploadIdRef.current = session.uploadId- setUploadSnapshot({- state: session.state === 'paused' ? 'paused' : 'uploading',+ const initialSnapshot: GfsUploadSnapshot = {+ state: session.state as GfsUploadSnapshot['state'],session,uploadedBytes: session.committedBytes,totalBytes: session.expectedBytes,- })+ }+ setUploadSnapshot(initialSnapshot)+ // The legacy compatibility path returns a completed resource receipt,+ // not a resumable upload session. There is no v2 snapshot to poll under+ // that resource id; refresh the folder and finish immediately.+ if (session.state === 'completed') {+ await refreshGfs()+ return initialSnapshot+ }const snapshot = await waitForUpload(session.uploadId)if (snapshot.state === 'completed') await refreshGfs()return snapshot@@ -328,12 +336,17 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {input.resumeUploadId)uploadIdRef.current = session.uploadId- setUploadSnapshot({- state: session.state === 'paused' ? 'paused' : 'uploading',+ const initialSnapshot: GfsUploadSnapshot = {+ state: session.state as GfsUploadSnapshot['state'],session,uploadedBytes: session.committedBytes,totalBytes: session.expectedBytes,- })+ }+ setUploadSnapshot(initialSnapshot)+ if (session.state === 'completed') {+ await refreshGfs()+ return initialSnapshot+ }const snapshot = await waitForUpload(session.uploadId)if (snapshot.state === 'completed') await refreshGfs()return snapshotdiff --git a/docs/architecture/gfs-upload-v2.md b/docs/architecture/gfs-upload-v2.mdindex 7448e4f1f..f519c4429 100644--- a/docs/architecture/gfs-upload-v2.md+++ b/docs/architecture/gfs-upload-v2.md@@ -39,6 +39,15 @@ finite and identical in Desktop and Control UI: `408`, `425`, `429`, `500`,`502`, `503`, and `504`. `507 Insufficient Storage` and every other `5xx` areterminal until an explicit server-side recovery/retry is requested. A servermay return `Retry-After`; clients honor it within their bounded retry budget.+For binary part writes only, a transport failure or `502/503/504` receives a+separate six-attempt service-recovery budget; ordinary lifecycle requests and+other retryable statuses retain the three-attempt budget. This covers a bounded+writer deployment restart without replaying a part blindly: each ambiguous+attempt still reconciles the durable status and checksum first.+The branch-owned Playwright gate can enable the negative journeys by setting+`GFS_UPLOAD_V2_NEGATIVE_E2E=1`; they remain opt-in because they mutate HCC,+restart the writer, and revoke only the seeded grant. The gate still requires+an explicit non-production context and profile-owned random URLs.## Edge admission limits@@ -69,6 +78,12 @@ is unavailable. A denied budget returns stable`429 { error: "gfs_upload_rate_limited", limit, retryAfterSeconds }` plus`Retry-After`; both relays preserve that contract.+The source-IP bucket is trusted only across the authenticated+`external-rest-api` service boundary. Control API ignores a client-supplied+`x-gfs-upload-source-ip` from any other caller and falls back to the socket or+trusted proxy address, so a valid internal token cannot rotate the IP key to+evade the per-IP budget.+## End-to-end shape```text@@ -122,6 +137,13 @@ false cancellation. The publication transaction re-checks state and epochbefore writing the resource, so a canceled response can never accompany apublished resource.+Client progress is monotonic for stale `uploading` snapshots that arrive out+of order. A failed part retry is different: its in-flight contribution is+removed so the next aggregate may move downward to the truthful committed+plus in-flight value. Completion is the only state that may expose the exact+file size; the browser and Desktop tests cover both the stale-snapshot guard+and this failure correction.+## Persistence and safetyDesktop upload persistence is a version-2 envelope. Every resumable record is@@ -133,11 +155,15 @@ deliberate team switch, or runtime/API-base switch blocks new dispatch, advancesthe epoch, aborts and awaits active v2 and legacy fallback work, clearscredentials, and persists the old records as local `suspended_auth`. A finiteinternal team-context hop temporarily blocks only new GFS dispatches while it-borrows the other team's token; existing uploads keep their captured token and-scope, and the original team scope is restored before the gate opens. Resume is-user-explicit and succeeds only when owner, team, environment, base URL, and-drive match exactly; the new auth epoch is bound only after the server session-is revalidated.+borrows the other team's token; read-only listing/snapshot and controls for an+existing job remain available, existing uploads keep their captured token and+scope, and the original team scope is restored before the gate opens. A+deliberate switch fences the old uploads after the replacement session+succeeds; if that local fence cannot be persisted, the app fails closed by+clearing the replacement session and credentials rather than leaving an+ambiguous team/token pairing. Resume is user-explicit and succeeds only when+owner, team, environment, base URL, and drive match exactly; the new auth epoch+is bound only after the server session is revalidated.The legacy compatibility fallback still accepts at most 16 MiB. It opens onedescriptor with no-follow semantics where available, validates the descriptordiff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex c589ae108..e7eecf31b 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.69",+ "version": "0.1.70","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.69",+ "version": "0.1.70","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex b98e1c008..fe0cdaf72 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.69",+ "version": "0.1.70","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.tsindex edf9c38a6..903f66fa4 100644--- a/external-rest-api/src/app.ts+++ b/external-rest-api/src/app.ts@@ -21,7 +21,8 @@ import { createExternalWorkflowsRouter } from './routes/workflows.js'// Upload v2 part bodies are streamed octets and must not pass through the// global JSON parser (which would buffer/reject the binary payload).-const GFS_UPLOAD_PART_PATH = /^\/api\/v1\/me\/gfs\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'+const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/parts/[0-9]+$`, 'i')export function createApp() {const app = express()diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.tsindex 8884ca6c1..40f76d293 100644--- a/gfs-controller/src/api/serve.ts+++ b/gfs-controller/src/api/serve.ts@@ -178,7 +178,7 @@ const ACCESSIBLE_RE = /^\/v1\/accessible$/;const COPY_RE = /^\/v1\/copy$/;const CAPABILITIES_RE = /^\/v1\/capabilities$/;const UPLOAD_COLLECTION_RE = /^\/v1\/uploads$/;-const UPLOAD_ITEM_RE = /^\/v1\/uploads\/([0-9a-f-]{36})(?:\/(status|parts\/([0-9]+)|pause|resume|complete))?$/i;+const UPLOAD_ITEM_RE = /^\/v1\/uploads\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\/(status|parts\/([0-9]+)|pause|resume|complete))?$/i;const GFS_URI_RE = /^gfs:\/\/([^/]+)\/([^/?#]+)$/;function bearerToken(req: IncomingMessage): string {diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.tsindex 72a3c2b9a..af5ae9a0e 100644--- a/gfs-controller/src/index.ts+++ b/gfs-controller/src/index.ts@@ -1,8 +1,7 @@-import { createHash, randomUUID } from "node:crypto";+import { randomUUID } from "node:crypto";import { constants } from "node:fs";import { lstat, mkdir, open, rm, stat } from "node:fs/promises";import { resolve } from "node:path";-import { Readable } from "node:stream";import { Client, Pool } from "pg";import { verifyGfsToken } from "./auth/verify";import { KeyConfigError, loadVerificationKey } from "./auth/keys";@@ -20,46 +19,14 @@ import { PgBlobStagingStore, reconcileExpiredBlobs } from "./db/blobStaging";import { GfsMetrics } from "./metrics";import { GfsServer, ReadinessDeps } from "./server";import { BlobStore } from "./storage/blobStore";-import { GfsUploadSessionService, uploadCapabilities, type UploadPartRow, type UploadSessionRow } from "./upload/uploadSession";+import { GfsUploadSessionService, uploadCapabilities } from "./upload/uploadSession";+import { createGfsUploadFinalizer } from "./upload/uploadFinalizer";import { GfsError } from "./api/errors";function errMsg(err: unknown): string {return err instanceof Error ? err.message : String(err);}-function orderedUploadStream(storageMountPath: string, parts: readonly UploadPartRow[], signal?: AbortSignal): Readable {- return Readable.from((async function* () {- for (const part of [...parts].sort((a, b) => a.partNumber - b.partNumber)) {- signal?.throwIfAborted();- const root = resolve(storageMountPath);- const expectedPath = resolve(root, ".uploads", part.uploadId, "parts", `${part.partNumber}.part`);- if (part.stagingPath !== expectedPath || !part.stagingPath.startsWith(`${root}/.uploads/${part.uploadId}/parts/`)) {- throw new GfsError("path_invalid", "staged upload part path is invalid");- }- const handle = await open(part.stagingPath, constants.O_RDONLY | constants.O_NOFOLLOW);- const buffer = Buffer.allocUnsafe(64 * 1024);- const digest = createHash("sha256");- let bytes = 0;- try {- for (;;) {- signal?.throwIfAborted();- const result = await handle.read(buffer, 0, buffer.length, null);- if (result.bytesRead === 0) break;- const chunk = Buffer.from(buffer.subarray(0, result.bytesRead));- bytes += result.bytesRead;- digest.update(chunk);- yield chunk;- }- } finally {- await handle.close().catch(() => undefined);- }- if (bytes !== part.lengthBytes || digest.digest("hex") !== part.sha256) {- throw new GfsError("checksum_mismatch", `staged upload part ${part.partNumber} failed verification`);- }- }- })());-}-async function assertUploadV2Ready(pool: Pool,storageMountPath: string,@@ -293,76 +260,13 @@ async function main(): Promise<void> {const decision = await permissions.authorize(targetContext, targetRid, "write");if (!decision.allowed) throw new GfsError("forbidden", `not authorized to ${operation} upload`);},- finalize: async (session: UploadSessionRow, parts: UploadPartRow[], signal?: AbortSignal, deadlineAtMs?: number) => {- const capturedPermissionEpoch = permissions.permissionEpoch();- const source = {- stream: orderedUploadStream(config.storageMountPath, parts, signal),- expectedBytes: session.expectedBytes,- ...(session.wholeSha256 ? { expectedSha256: session.wholeSha256 } : {}),- };- const mutation = { subject: session.primarySubject, requestId: session.uploadId, audit };- const onPublished = async (client: import("./db/writeStore").TxClient, published: import("./api/read").GfsResource, publicationSignal?: AbortSignal): Promise<void> => {- publicationSignal?.throwIfAborted();- if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {- throw new GfsError("precondition_failed", "upload finalization timed out");- }- const currentPermissionEpoch = permissions.permissionEpoch();- if (- capturedPermissionEpoch.bypassed- || currentPermissionEpoch.bypassed- || currentPermissionEpoch.generation !== capturedPermissionEpoch.generation- ) {- throw new GfsError("precondition_failed", "authorization changed during upload finalization");- }- const targetRid = session.operation === "create" ? session.parentRid : session.resourceRid;- if (!targetRid) throw new GfsError("path_invalid", "upload target is required");- const targetContext = await resolveAuthzContext(pool, {- sub: session.ownerSubject,- drive: session.drive,- });- const decision = await permissions.authorize(targetContext, targetRid, "write");- publicationSignal?.throwIfAborted();- if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {- throw new GfsError("precondition_failed", "upload finalization timed out");- }- if (!decision.allowed) throw new GfsError("forbidden", "upload authorization was revoked before publication");- const completed = await client.query(- `UPDATE gfs_upload_sessions- SET state = 'completed', result_resource_id = $2, result_version = $3,- result_sha256 = $4, completed_at = now(), updated_at = now()- WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $5- RETURNING upload_id`,- [session.uploadId, published.resourceId, published.version, published.contentSha256, session.sessionEpoch]- );- publicationSignal?.throwIfAborted();- if (completed.rows.length !== 1) throw new GfsError("upload_aborted", "upload session changed before publication");- };- const resource = session.operation === "create"- ? await writeService.create({- drive: session.drive,- parentId: session.parentRid!,- name: session.resourceName!,- kind: "file",- resourceId: session.uploadId,- content: source,- mutation,- onPublished,- signal,- deadlineAtMs,- })- : await writeService.replace({- drive: session.drive,- resourceId: session.resourceRid!,- ifMatch: session.ifMatch ?? undefined,- content: source,- mutation,- onPublished,- signal,- deadlineAtMs,- });- if (!resource.contentSha256) throw new GfsError("internal", "published upload has no checksum");- return { resourceId: resource.resourceId, version: resource.version, sha256: resource.contentSha256 };- },+ finalize: createGfsUploadFinalizer({+ pool,+ storageMountPath: config.storageMountPath,+ permissions,+ writeService,+ audit,+ }),})),} : {}),});diff --git a/gfs-controller/src/upload/protocol.test.ts b/gfs-controller/src/upload/protocol.test.tsindex 2a0b47352..f10911d68 100644--- a/gfs-controller/src/upload/protocol.test.ts+++ b/gfs-controller/src/upload/protocol.test.ts@@ -1,13 +1,13 @@import { describe, expect, it } from 'vitest'import {- disabledGfsUploadV2Capability,GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,GFS_UPLOAD_V2_FALLBACK_CONCURRENCY,+ GFS_UPLOAD_V2_INSTABILITY_FAILURE_THRESHOLD,GFS_UPLOAD_V2_MAX_PART_BYTES,+ GFS_UPLOAD_V2_PREFERRED_PART_BYTES,GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES,- GFS_UPLOAD_V2_PREFERRED_PART_BYTES,- GFS_UPLOAD_V2_INSTABILITY_FAILURE_THRESHOLD,+ disabledGfsUploadV2Capability,isGfsUploadV2Route,partCountFor,partGeometry,@@ -32,9 +32,16 @@ describe('GFS Upload v2 frozen contract', () => {})it('calculates exact 8 MiB geometry at the product boundary', () => {- const geometry = { expectedBytes: GFS_UPLOAD_V2_PRODUCT_MAX_BYTES, partBytes: GFS_UPLOAD_V2_PREFERRED_PART_BYTES }+ const geometry = {+ expectedBytes: GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,+ partBytes: GFS_UPLOAD_V2_PREFERRED_PART_BYTES,+ }expect(partCountFor(geometry)).toBe(25)- expect(partGeometry(geometry, 0)).toEqual({ partNumber: 0, offsetBytes: 0, lengthBytes: 8_388_608 })+ expect(partGeometry(geometry, 0)).toEqual({+ partNumber: 0,+ offsetBytes: 0,+ lengthBytes: 8_388_608,+ })expect(partGeometry(geometry, 24)).toEqual({partNumber: 24,offsetBytes: 201_326_592,@@ -55,11 +62,40 @@ describe('GFS Upload v2 frozen contract', () => {)})+ it.each([+ [0, 1 * 1024 * 1024],+ [1, 1 * 1024 * 1024],+ [1_048_577, 1 * 1024 * 1024],+ [16 * 1024 * 1024 + 1, 8 * 1024 * 1024],+ [209_715_200, 16 * 1024 * 1024],+ ])('keeps generated geometry contiguous for expected=%d part=%d', (expectedBytes, partBytes) => {+ const geometry = { expectedBytes, partBytes }+ const count = partCountFor(geometry)+ const parts = Array.from({ length: count }, (_, partNumber) =>+ partGeometry(geometry, partNumber)+ )+ expect(parts.reduce((sum, part) => sum + part.lengthBytes, 0)).toBe(expectedBytes)+ for (let index = 0; index < parts.length; index += 1) {+ const part = parts[index]!+ expect(part.offsetBytes).toBe(+ index === 0 ? 0 : parts[index - 1]!.offsetBytes + parts[index - 1]!.lengthBytes+ )+ expect(part.lengthBytes).toBeGreaterThan(0)+ expect(part.lengthBytes).toBeLessThanOrEqual(partBytes)+ validatePartGeometry(geometry, part)+ }+ })+it('recognizes only the exact v2 routes', () => {expect(isGfsUploadV2Route('/v1/capabilities')).toBe(true)expect(isGfsUploadV2Route('/v1/uploads')).toBe(true)- expect(isGfsUploadV2Route('/v1/uploads/01234567-89ab-cdef-0123-456789abcdef/parts/0')).toBe(true)+ expect(isGfsUploadV2Route('/v1/uploads/01234567-89ab-cdef-0123-456789abcdef/parts/0')).toBe(+ true+ )expect(isGfsUploadV2Route('/v1/resources/0123456789abcdef0123456789abcdef')).toBe(false)expect(isGfsUploadV2Route('/v1/uploads/not-an-id/parts/0')).toBe(false)+ expect(isGfsUploadV2Route('/v1/uploads/------------------------------------/parts/0')).toBe(+ false+ )})})diff --git a/gfs-controller/src/upload/protocol.ts b/gfs-controller/src/upload/protocol.tsindex 06cbc95d3..4be93c216 100644--- a/gfs-controller/src/upload/protocol.ts+++ b/gfs-controller/src/upload/protocol.ts@@ -1,10 +1,12 @@/*** Shared, side-effect-free constants and geometry helpers for GFS Upload v2.*- * The writer and both clients import this module so the capability response,- * session admission, part routes, and UI geometry share one arithmetic- * definition. The route remains capability-gated: disabled deployments still- * advertise the legacy surface and reject v2 mutations.+ * The writer owns the authoritative arithmetic used by the capability+ * response, session admission, and part routes. The separately packaged web+ * and Desktop clients mirror the wire values at their boundaries rather than+ * importing writer code; contract tests pin those copies to this definition.+ * The route remains capability-gated: disabled deployments still advertise the+ * legacy surface and reject v2 mutations.*/export const GFS_UPLOAD_V2_PRODUCT_MAX_BYTES = 209_715_200@@ -23,6 +25,8 @@ export const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000export const GFS_UPLOAD_V2_FINALIZE_TIMEOUT_MS = 600_000export const GFS_UPLOAD_V2_STALE_PART_LEASE_MS = 600_000+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'+export type GfsUploadOperation = 'create' | 'replace'export interface GfsUploadGeometry {@@ -63,7 +67,11 @@ export function partCountFor({ expectedBytes, partBytes }: GfsUploadGeometry): n}export function validateGeometry({ expectedBytes, partBytes }: GfsUploadGeometry): void {- if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0 || expectedBytes > GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES) {+ if (+ !Number.isSafeInteger(expectedBytes) ||+ expectedBytes < 0 ||+ expectedBytes > GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES+ ) {throw new GfsUploadGeometryError('invalid_expected_bytes',`expectedBytes must be an integer from 0 through ${GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES}`@@ -113,13 +121,22 @@ export function validatePartGeometry(part: GfsUploadPartGeometry): void {if (!Number.isSafeInteger(part.partNumber) || part.partNumber < 0) {- throw new GfsUploadGeometryError('invalid_part_number', 'partNumber must be a non-negative integer')+ throw new GfsUploadGeometryError(+ 'invalid_part_number',+ 'partNumber must be a non-negative integer'+ )}if (!Number.isSafeInteger(part.offsetBytes) || part.offsetBytes < 0) {- throw new GfsUploadGeometryError('invalid_part_offset', 'offsetBytes must be a non-negative integer')+ throw new GfsUploadGeometryError(+ 'invalid_part_offset',+ 'offsetBytes must be a non-negative integer'+ )}if (!Number.isSafeInteger(part.lengthBytes) || part.lengthBytes <= 0) {- throw new GfsUploadGeometryError('invalid_part_length', 'lengthBytes must be a positive integer')+ throw new GfsUploadGeometryError(+ 'invalid_part_length',+ 'lengthBytes must be a positive integer'+ )}const expected = partGeometry(geometry, part.partNumber)if (part.offsetBytes !== expected.offsetBytes) {@@ -141,7 +158,12 @@ export function isGfsUploadV2Route(pathname: string): boolean {return (pathname === '/v1/capabilities' ||pathname === '/v1/uploads' ||- /^\/v1\/uploads\/[0-9a-f-]{36}(?:\/status|\/parts\/[0-9]+|\/pause|\/resume|\/complete)?$/.test(pathname)+ new RegExp(+ `^/v1/uploads/${UUID_PATH}(?:/status|/parts/[0-9]+|/pause|/resume|/complete)?$`,+ 'i'+ ).test(+ pathname+ ))}diff --git a/gfs-controller/src/upload/uploadFinalizer.test.ts b/gfs-controller/src/upload/uploadFinalizer.test.tsnew file mode 100644index 000000000..91bef2e24--- /dev/null+++ b/gfs-controller/src/upload/uploadFinalizer.test.ts@@ -0,0 +1,171 @@+import { describe, expect, it, vi } from 'vitest'+import { createHash } from 'node:crypto'+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import type { DbAuditSink, PermissionClient } from '../authz/permissionClient'+import { createGfsUploadFinalizer } from './uploadFinalizer'+import type { UploadPartRow, UploadSessionRow } from './uploadSession'++const UPLOAD_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'+const PARENT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'+const BODY = Buffer.from('published upload')+const SHA256 = createHash('sha256').update(BODY).digest('hex')++function session(): UploadSessionRow {+ return {+ uploadId: UPLOAD_ID,+ idempotencyKey: 'finalizer-test',+ drive: 'main',+ ownerSubject: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',+ primarySubject: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',+ operation: 'create',+ requestFingerprint: 'fingerprint',+ parentRid: PARENT_ID,+ resourceRid: null,+ resourceName: 'payload.bin',+ ifMatch: null,+ expectedBytes: BODY.length,+ partBytes: BODY.length,+ partCount: 1,+ wholeSha256: SHA256,+ committedBytes: BODY.length,+ contiguousBytes: BODY.length,+ committedPartCount: 1,+ activePartCount: 0,+ sessionEpoch: 0,+ state: 'finalizing',+ resultResourceId: null,+ resultVersion: null,+ resultSha256: null,+ failureCode: null,+ expiresAt: new Date(Date.now() + 60_000).toISOString(),+ }+}++function part(root: string): UploadPartRow {+ return {+ uploadId: UPLOAD_ID,+ partNumber: 0,+ offsetBytes: 0,+ lengthBytes: BODY.length,+ sha256: SHA256,+ state: 'committed',+ stagingPath: join(root, '.uploads', UPLOAD_ID, 'parts', '0.part'),+ leaseEpoch: 0,+ }+}++function finalizerHarness(+ root: string,+ permissionEpoch: () => { generation: number; bypassed: boolean }+) {+ const permissions = {+ permissionEpoch,+ authorize: vi.fn().mockResolvedValue({ allowed: true }),+ } as unknown as PermissionClient+ const client = {+ query: vi.fn().mockResolvedValue({ rows: [{}] }),+ }+ const resource = {+ resourceId: UPLOAD_ID,+ rid: UPLOAD_ID.replaceAll('-', ''),+ drive: 'main',+ parentResourceId: PARENT_ID,+ name: 'payload.bin',+ kind: 'file',+ bytes: BODY.length,+ version: 0,+ blobKey: 'generation/payload',+ contentSha256: SHA256,+ deletedAt: null,+ path: '/payload.bin',+ updatedAt: new Date().toISOString(),+ }+ const writeService = {+ create: vi.fn(async (input: any) => {+ const chunks: Buffer[] = []+ for await (const chunk of input.content.stream) chunks.push(Buffer.from(chunk))+ expect(Buffer.concat(chunks)).toEqual(BODY)+ await input.onPublished(client, resource)+ return resource+ }),+ replace: vi.fn(),+ }+ const pool = { query: vi.fn().mockResolvedValue({ rows: [] }) }+ const finalizer = createGfsUploadFinalizer({+ pool: pool as never,+ storageMountPath: root,+ permissions,+ writeService: writeService as never,+ audit: {} as DbAuditSink,+ })+ return { finalizer, permissions, client, writeService }+}++describe('production GFS upload finalizer seam', () => {+ it('streams committed parts and marks the session completed in the publication transaction', async () => {+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-'))+ try {+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })+ await writeFile(partPath, BODY)+ const harness = finalizerHarness(root, () => ({ generation: 1, bypassed: false }))++ await expect(harness.finalizer(session(), [part(root)])).resolves.toEqual({+ resourceId: UPLOAD_ID,+ version: 0,+ sha256: SHA256,+ })+ expect(harness.client.query).toHaveBeenCalledWith(+ expect.stringContaining("SET state = 'completed'"),+ expect.arrayContaining([UPLOAD_ID, UPLOAD_ID, 0, SHA256, 0])+ )+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })++ it('rejects a permission-epoch change before publication and never marks completion', async () => {+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-revoked-'))+ try {+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })+ await writeFile(partPath, BODY)+ let generation = 1+ const harness = finalizerHarness(root, () => {+ return { generation, bypassed: false }+ })+ harness.permissions.authorize = vi.fn().mockImplementation(async () => {+ generation = 2+ return { allowed: true }+ }) as never++ await expect(harness.finalizer(session(), [part(root)])).rejects.toMatchObject({+ code: 'precondition_failed',+ message: 'authorization changed during upload finalization',+ })+ expect(harness.client.query).not.toHaveBeenCalled()+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })++ it('rejects an explicit authorization denial before publication', async () => {+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-denied-'))+ try {+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })+ await writeFile(partPath, BODY)+ const harness = finalizerHarness(root, () => ({ generation: 1, bypassed: false }))+ harness.permissions.authorize = vi.fn().mockResolvedValue({ allowed: false }) as never++ await expect(harness.finalizer(session(), [part(root)])).rejects.toMatchObject({+ code: 'forbidden',+ })+ expect(harness.client.query).not.toHaveBeenCalled()+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })+})diff --git a/gfs-controller/src/upload/uploadFinalizer.ts b/gfs-controller/src/upload/uploadFinalizer.tsnew file mode 100644index 000000000..a25009c6f--- /dev/null+++ b/gfs-controller/src/upload/uploadFinalizer.ts@@ -0,0 +1,193 @@+import { createHash } from 'node:crypto'+import { constants } from 'node:fs'+import { open } from 'node:fs/promises'+import { resolve } from 'node:path'+import { Readable } from 'node:stream'+import type { Pool } from 'pg'+import { GfsError } from '../api/errors'+import type { GfsResource } from '../api/read'+import type { DbAuditSink, PermissionClient } from '../authz/permissionClient'+import { resolveAuthzContext } from '../authz/subjectResolver'+import { GfsWriteService, type TxClient } from '../db/writeStore'+import type { UploadPartRow, UploadSessionRow } from './uploadSession'++function orderedUploadStream(+ storageMountPath: string,+ parts: readonly UploadPartRow[],+ signal?: AbortSignal+): Readable {+ return Readable.from(+ (async function* () {+ for (const part of [...parts].sort((a, b) => a.partNumber - b.partNumber)) {+ signal?.throwIfAborted()+ const root = resolve(storageMountPath)+ const expectedPath = resolve(+ root,+ '.uploads',+ part.uploadId,+ 'parts',+ `${part.partNumber}.part`+ )+ if (+ part.stagingPath !== expectedPath ||+ !part.stagingPath.startsWith(`${root}/.uploads/${part.uploadId}/parts/`)+ ) {+ throw new GfsError('path_invalid', 'staged upload part path is invalid')+ }+ const handle = await open(part.stagingPath, constants.O_RDONLY | constants.O_NOFOLLOW)+ const buffer = Buffer.allocUnsafe(64 * 1024)+ const digest = createHash('sha256')+ let bytes = 0+ try {+ for (;;) {+ signal?.throwIfAborted()+ const result = await handle.read(buffer, 0, buffer.length, null)+ if (result.bytesRead === 0) break+ const chunk = Buffer.from(buffer.subarray(0, result.bytesRead))+ bytes += result.bytesRead+ digest.update(chunk)+ yield chunk+ }+ } finally {+ await handle.close().catch(() => undefined)+ }+ if (bytes !== part.lengthBytes || digest.digest('hex') !== part.sha256) {+ throw new GfsError(+ 'checksum_mismatch',+ `staged upload part ${part.partNumber} failed verification`+ )+ }+ }+ })()+ )+}++export interface GfsUploadFinalizerDeps {+ pool: Pool+ storageMountPath: string+ permissions: PermissionClient+ writeService: Pick<GfsWriteService, 'create' | 'replace'>+ audit: DbAuditSink+}++/**+ * Production upload finalizer seam. Keeping this closure in a small module+ * makes the writer's stream, permission-epoch recheck, and same-transaction+ * receipt update executable in focused tests instead of hiding them behind+ * the process bootstrap.+ */+export function createGfsUploadFinalizer(+ deps: GfsUploadFinalizerDeps+): (+ session: UploadSessionRow,+ parts: UploadPartRow[],+ signal?: AbortSignal,+ deadlineAtMs?: number+) => Promise<{ resourceId: string; version: number; sha256: string }> {+ return async (session, parts, signal, deadlineAtMs) => {+ const capturedPermissionEpoch = deps.permissions.permissionEpoch()+ const source = {+ stream: orderedUploadStream(deps.storageMountPath, parts, signal),+ expectedBytes: session.expectedBytes,+ ...(session.wholeSha256 ? { expectedSha256: session.wholeSha256 } : {}),+ }+ const mutation = {+ subject: session.primarySubject,+ requestId: session.uploadId,+ audit: deps.audit,+ }+ const onPublished = async (+ client: TxClient,+ published: GfsResource,+ publicationSignal?: AbortSignal+ ): Promise<void> => {+ publicationSignal?.throwIfAborted()+ if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {+ throw new GfsError('precondition_failed', 'upload finalization timed out')+ }+ const currentPermissionEpoch = deps.permissions.permissionEpoch()+ if (+ capturedPermissionEpoch.bypassed ||+ currentPermissionEpoch.bypassed ||+ currentPermissionEpoch.generation !== capturedPermissionEpoch.generation+ ) {+ throw new GfsError(+ 'precondition_failed',+ 'authorization changed during upload finalization'+ )+ }+ const targetRid = session.operation === 'create' ? session.parentRid : session.resourceRid+ if (!targetRid) throw new GfsError('path_invalid', 'upload target is required')+ const targetContext = await resolveAuthzContext(deps.pool, {+ sub: session.ownerSubject,+ drive: session.drive,+ })+ const decision = await deps.permissions.authorize(targetContext, targetRid, 'write')+ publicationSignal?.throwIfAborted()+ if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {+ throw new GfsError('precondition_failed', 'upload finalization timed out')+ }+ const postAuthorizationEpoch = deps.permissions.permissionEpoch()+ if (+ postAuthorizationEpoch.bypassed ||+ postAuthorizationEpoch.generation !== capturedPermissionEpoch.generation+ ) {+ throw new GfsError(+ 'precondition_failed',+ 'authorization changed during upload finalization'+ )+ }+ if (!decision.allowed) {+ throw new GfsError('forbidden', 'upload authorization was revoked before publication')+ }+ const completed = await client.query(+ `UPDATE gfs_upload_sessions+ SET state = 'completed', result_resource_id = $2, result_version = $3,+ result_sha256 = $4, completed_at = now(), updated_at = now()+ WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $5+ RETURNING upload_id`,+ [+ session.uploadId,+ published.resourceId,+ published.version,+ published.contentSha256,+ session.sessionEpoch,+ ]+ )+ publicationSignal?.throwIfAborted()+ if (completed.rows.length !== 1) {+ throw new GfsError('upload_aborted', 'upload session changed before publication')+ }+ }+ const resource =+ session.operation === 'create'+ ? await deps.writeService.create({+ drive: session.drive,+ parentId: session.parentRid!,+ name: session.resourceName!,+ kind: 'file',+ resourceId: session.uploadId,+ content: source,+ mutation,+ onPublished,+ signal,+ deadlineAtMs,+ })+ : await deps.writeService.replace({+ drive: session.drive,+ resourceId: session.resourceRid!,+ ifMatch: session.ifMatch ?? undefined,+ content: source,+ mutation,+ onPublished,+ signal,+ deadlineAtMs,+ })+ if (!resource.contentSha256) throw new GfsError('internal', 'published upload has no checksum')+ return {+ resourceId: resource.resourceId,+ version: resource.version,+ sha256: resource.contentSha256,+ }+ }+}diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex bb561f9ee..6b0476048 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -697,6 +697,18 @@ describe('GfsUploadSessionService', () => {(uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)+ // The first reconciliation attempt must exercise the terminal-session+ // cancellation fence, then release it even though filesystem cleanup is+ // still blocked by the non-directory path. This catches a vacuous+ // assertion before reconcile() and proves the fence is bounded across+ // repeated cleanup cycles.+ await expect(uploads.reconcile()).resolves.toEqual({ staleParts: 0, expiredSessions: 0 })+ expect(+ (uploads as unknown as { canceledUploads: Set<string> }).canceledUploads.has(+ created.session.uploadId+ )+ ).toBe(false)+await rm(uploadDirectory, { force: true })await mkdir(uploadDirectory, { recursive: true })await writeFile(join(uploadDirectory, 'finalizer-marker'), 'pending')diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex e094cfb0a..c0d8e73f8 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -1354,6 +1354,12 @@ export class GfsUploadSessionService {)}).catch(() => undefined)+ // A terminal session with no registered flight no longer needs the+ // in-memory cancellation fence. Keep it only while an operation is+ // still alive so repeated filesystem failures cannot leak one set+ // entry per terminal upload; the durable terminal row remains the+ // authority for the next reconciliation cycle.+ if (!this.hasInFlight(uploadId)) this.canceledUploads.delete(uploadId)}}return { staleParts, expiredSessions }diff --git a/scripts/e2e/gfs-upload-v2-gate.sh b/scripts/e2e/gfs-upload-v2-gate.shindex 7665f9ac4..769bd38b6 100755--- a/scripts/e2e/gfs-upload-v2-gate.sh+++ b/scripts/e2e/gfs-upload-v2-gate.sh@@ -26,22 +26,37 @@ python3 tools/e2e_static_audit.py \run_ui="${GFS_UPLOAD_V2_RUN_UI:-0}"run_desktop="${GFS_UPLOAD_V2_RUN_DESKTOP:-0}"run_integration="${GFS_UPLOAD_V2_RUN_INTEGRATION:-0}"+# Negative journeys mutate HCC, restart the writer, and revoke only the seeded+# grant. Keep them explicit so a caller cannot turn an arbitrary context into+# a destructive default by selecting a UI lane alone.+run_negative="${GFS_UPLOAD_V2_NEGATIVE_E2E:-0}"if [[ "$run_ui" == "1" ]]; then- GFS_UPLOAD_V2_E2E=1 npm --prefix control-ui exec playwright test \+ GFS_UPLOAD_V2_E2E=1 GFS_UPLOAD_V2_NEGATIVE_E2E="$run_negative" npm --prefix control-ui exec playwright test \--config=playwright.config.ts --project=large-upload e2e/gfs-upload-v2.spec.tsfiif [[ "$run_desktop" == "1" ]]; then- GFS_UPLOAD_V2_E2E=1 npm --prefix desktop-app exec playwright test \+ GFS_UPLOAD_V2_E2E=1 GFS_UPLOAD_V2_NEGATIVE_E2E="$run_negative" npm --prefix desktop-app exec playwright test \--config=test/e2e-playwright/playwright.config.ts \--project=packaged-gfs-upload-v2 test/e2e-playwright/gfs-upload-v2.test.tsfiif [[ "$run_integration" == "1" ]]; then: "${CONTROL_API_REAL_PG_ADMIN_URL:?Set CONTROL_API_REAL_PG_ADMIN_URL for the real-Postgres integration lane}"- npm --prefix control-api test -- --run \- test/gfsUploadSession.realPostgres.integration.test.ts+ real_pg_log="$(mktemp)"+ real_pg_normalized="$(mktemp)"+ trap 'rm -f "$real_pg_log" "$real_pg_normalized"' EXIT+ npm --prefix control-api test -- --run realPostgres 2>&1 | tee "$real_pg_log"+ sed $'s/\033\\[[0-9;]*m//g' "$real_pg_log" >"$real_pg_normalized"+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$real_pg_normalized" || {+ echo 'real-Postgres E2E lane executed zero passing test files' >&2+ exit 1+ }+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$real_pg_normalized" || {+ echo 'real-Postgres E2E lane executed zero passing tests' >&2+ exit 1+ }# Runtime restart/chaos/load is caller-owned because it must be bound to the# exact branch profile and Cloudflare-backed dev hostname. Do not silently# substitute an older RED gate or report integration-only evidence as T2.diff --git a/scripts/tests/test-gfs-upload-v2-geometry-contract.sh b/scripts/tests/test-gfs-upload-v2-geometry-contract.shnew file mode 100755index 000000000..b76ba3223--- /dev/null+++ b/scripts/tests/test-gfs-upload-v2-geometry-contract.sh@@ -0,0 +1,28 @@+#!/usr/bin/env bash+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+cd "$ROOT"++fail() { echo "FAIL: $*" >&2; exit 1; }++read_const() {+ local file="$1" name="$2"+ sed -nE "s/^(export )?const ${name} = (.*);?$/\\2/p" "$file" \+ | head -1 \+ | tr -d '[:space:]'+}++gfsc_preferred="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"+gfsc_max="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"+ui_preferred="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_PREFERRED_PART_BYTES)"+ui_max="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_MAX_PART_BYTES)"+desktop_preferred="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"+desktop_max="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"++[[ -n "$gfsc_preferred" && "$gfsc_preferred" == "$ui_preferred" && "$gfsc_preferred" == "$desktop_preferred" ]] \+ || fail "preferred part geometry diverges between writer, Control UI, and Desktop"+[[ -n "$gfsc_max" && "$gfsc_max" == "$ui_max" && "$gfsc_max" == "$desktop_max" ]] \+ || fail "maximum part geometry diverges between writer, Control UI, and Desktop"++echo "PASS: GFS Upload v2 part geometry is aligned across writer, Control UI, and Desktop"diff --git a/tests/e2e/gfsUploadV2Fixtures.ts b/tests/e2e/gfsUploadV2Fixtures.tsindex 170cd6f99..68add8a36 100644--- a/tests/e2e/gfsUploadV2Fixtures.ts+++ b/tests/e2e/gfsUploadV2Fixtures.ts@@ -44,6 +44,30 @@ export async function createDiskUploadFixture(}}+/**+ * Creates the single-byte-over-limit fixture used by the opt-in negative+ * browser journey. Keep the normal fixture helper fail-closed at the product+ * ceiling so a positive test can never accidentally become an oversize test.+ */+export async function createOversizedDiskUploadFixture(+ extension = '.parquet',+ label = 'gfs-v2-oversize'+): Promise<DiskUploadFixture> {+ const byteLength = GFS_UPLOAD_V2_MAX_BYTES + 1+ const directory = await mkdtemp(path.join(os.tmpdir(), 'evenfire-gfs-upload-v2-'))+ const fileName = `${label}-${byteLength}${extension}`+ const filePath = path.join(directory, fileName)+ await writeFile(filePath, Buffer.alloc(0))+ await truncate(filePath, byteLength)+ return {+ directory,+ filePath,+ fileName,+ byteLength,+ sha256: await sha256File(filePath),+ }+}+export async function sha256File(filePath: string): Promise<string> {const hash = createHash('sha256')for await (const chunk of createReadStream(filePath)) hash.update(chunk as Buffer)diff --git a/tests/e2e/gfsUploadV2Runtime.ts b/tests/e2e/gfsUploadV2Runtime.tsnew file mode 100644index 000000000..86ae738e8--- /dev/null+++ b/tests/e2e/gfsUploadV2Runtime.ts@@ -0,0 +1,342 @@+import { execFile, execFileSync } from 'node:child_process'+import { createHash } from 'node:crypto'+import { existsSync, readFileSync } from 'node:fs'+import { homedir } from 'node:os'+import { resolve } from 'node:path'+import { promisify } from 'node:util'+import { firstDataLine, kubectlContext, runControlPostgresSql, sqlLiteral } from './gfsFixtureCore'++const execFileAsync = promisify(execFile)+const HCC_NAMESPACE = 'control-plane'+const HCC_DEPLOYMENT = 'host-context-controller'+const GFS_NAMESPACE = 'gfs'+const GFS_WRITER_DEPLOYMENT = 'gfsc-writer'+const GFS_READER_DEPLOYMENT = 'gfsc-reader'+const GFS_V2_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_V2_ENABLED'+const GFS_V2_WRITER_ENV = 'GFS_UPLOAD_V2_ENABLED'+const BRANCH_CONTEXT_RE = /^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/++export interface GfsRuntimeImageMarker {+ clusterFingerprint?: string+ imageSource?: string+ imageTag?: string+ imagesGeneratedAt?: string+}++export interface ExpectedGfsRuntimeImageMarker {+ imageSource?: string+ imageTag?: string+ clusterFingerprint?: string+ imagesGeneratedAt?: string+}++/**+ * Local Minikube image mode intentionally has no registry tag. GHCR mode must+ * carry one because it is the immutable image coordinate that proves which+ * artifact was deployed. Both modes still require the marker's source and+ * acquisition timestamp so an empty local tag cannot be confused with a+ * missing marker field.+ */+export function validateGfsRuntimeImageMarker(+ marker: GfsRuntimeImageMarker,+ expected: ExpectedGfsRuntimeImageMarker = {}+): void {+ if (+ !marker.clusterFingerprint ||+ !marker.imageSource ||+ typeof marker.imageTag !== 'string' ||+ !marker.imagesGeneratedAt+ ) {+ throw new Error(+ 'refusing GFS E2E mutation: pre-gate marker lacks cluster fingerprint, image coordinates, or acquisition stamp'+ )+ }+ if (marker.imageSource !== 'local' && marker.imageSource !== 'ghcr') {+ throw new Error(`refusing GFS E2E mutation: unsupported imageSource ${marker.imageSource}`)+ }+ if (marker.imageSource === 'ghcr' && marker.imageTag.length === 0) {+ throw new Error(+ 'refusing GFS E2E mutation: GHCR image marker is missing its immutable imageTag'+ )+ }+ if (expected.imageSource && marker.imageSource !== expected.imageSource) {+ throw new Error(+ 'refusing GFS E2E mutation: imageSource does not match the caller-owned profile'+ )+ }+ if (expected.imageTag !== undefined && marker.imageTag !== expected.imageTag) {+ throw new Error('refusing GFS E2E mutation: imageTag does not match the caller-owned profile')+ }+ if (expected.clusterFingerprint && marker.clusterFingerprint !== expected.clusterFingerprint) {+ throw new Error(+ 'refusing GFS E2E mutation: clusterFingerprint does not match the caller-owned profile'+ )+ }+ if (expected.imagesGeneratedAt && marker.imagesGeneratedAt !== expected.imagesGeneratedAt) {+ throw new Error(+ 'refusing GFS E2E mutation: imagesGeneratedAt does not match the caller-owned profile'+ )+ }+}++function profilePorts(context: string): Map<string, string> {+ const path =+ process.env.CLERUM_PROFILE_PORTS_ENV ||+ process.env.E2E_PROFILE_PORTS_ENV ||+ resolve(homedir(), '.cache/clerum/minikube-profiles', context, 'ports.env')+ if (!existsSync(path)) {+ throw new Error(`refusing GFS E2E mutation: profile ports.env is missing at ${path}`)+ }+ const values = new Map<string, string>()+ for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {+ const match = /^(?:export\s+)?([A-Z0-9_]+)=(.*)$/.exec(line.trim())+ if (match) values.set(match[1]!, match[2]!.replace(/^['"]|['"]$/g, ''))+ }+ return values+}++function normalizeUrl(value: string): string {+ return value.replace(/\/$/, '')+}++/**+ * Mutating negative journeys are allowed only on the exact branch profile+ * whose generated port map is present in the caller environment. A context+ * name alone is not ownership proof: another healthy Minikube profile could+ * otherwise receive an HCC mutation or writer restart.+ */+async function assertOwnedRuntimeContext(): Promise<void> {+ const context = kubectlContext()+ if (/prod|production/i.test(context)) {+ throw new Error(`refusing GFS E2E mutation on production-like context ${context}`)+ }+ const explicitlyAllowed = new Set(+ (process.env.E2E_ALLOWED_CONTEXTS || '')+ .split(',')+ .map(value => value.trim())+ .filter(Boolean)+ )+ if (!BRANCH_CONTEXT_RE.test(context) && !explicitlyAllowed.has(context)) {+ throw new Error(+ `refusing GFS E2E mutation on unscoped context ${context}; use a generated branch profile or E2E_ALLOWED_CONTEXTS`+ )+ }+ const ports = profilePorts(context)+ const required = [+ ['CONTROL_UI_BASE_URL', 'CONTROL_UI_URL'],+ ['EXTERNAL_REST_API_BASE_URL', 'EXTERNAL_REST_API_URL'],+ ] as const+ for (const [actualKey, profileKey] of required) {+ const actual = process.env[actualKey] || process.env[actualKey.replace('_BASE_URL', '_URL')]+ const expected = ports.get(profileKey)+ if (!expected || !actual || normalizeUrl(actual) !== normalizeUrl(expected)) {+ throw new Error(+ `refusing GFS E2E mutation: ${actualKey} does not match ${profileKey} in profile ports.env`+ )+ }+ }+ const rpcActual = process.env.RPC_PROXY_BASE_URL || process.env.RPC_PROXY_URL+ if (rpcActual) {+ const rpcExpected = ports.get('RPC_PROXY_URL')+ if (!rpcExpected || normalizeUrl(rpcActual) !== normalizeUrl(rpcExpected)) {+ throw new Error(+ 'refusing GFS E2E mutation: RPC_PROXY_BASE_URL does not match RPC_PROXY_URL in profile ports.env'+ )+ }+ }++ const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {+ cwd: process.cwd(),+ encoding: 'utf8',+ }).trim()+ const dirty = execFileSync('git', ['status', '--porcelain', '--untracked-files=normal'], {+ cwd: repoRoot,+ encoding: 'utf8',+ }).trim()+ if (dirty) {+ throw new Error(+ 'refusing GFS E2E mutation: the worktree is dirty; commit and run pre-gate-sync before fault injection'+ )+ }+ const expectedWorktreeId = createHash('sha1').update(repoRoot).digest('hex')+ const expectedHead = execFileSync('git', ['rev-parse', 'HEAD'], {+ cwd: repoRoot,+ encoding: 'utf8',+ }).trim()+ const marker = JSON.parse(+ await kubectl([+ '-n',+ HCC_NAMESPACE,+ 'get',+ 'configmap/clerum-pre-gate-sync-state',+ '-o',+ 'json',+ ])+ ) as {+ data?: {+ worktreeId?: string+ gitHead?: string+ clusterFingerprint?: string+ imageSource?: string+ imageTag?: string+ imagesGeneratedAt?: string+ }+ }+ if (marker.data?.worktreeId !== expectedWorktreeId) {+ throw new Error(+ 'refusing GFS E2E mutation: cluster ownership marker does not match this worktree; run pre-gate-sync first'+ )+ }+ if (marker.data?.gitHead !== expectedHead) {+ throw new Error(+ `refusing GFS E2E mutation: cluster gitHead ${marker.data?.gitHead || '<missing>'} does not match ${expectedHead}`+ )+ }+ const markerData = marker.data+ const expectedImageSource = process.env.E2E_IMAGE_SOURCE || process.env.IMAGE_SOURCE || 'local'+ const expectedImageTag = process.env.E2E_IMAGE_TAG || process.env.IMAGE_TAG+ const expectedClusterFingerprint =+ process.env.E2E_EXPECTED_CLUSTER_FINGERPRINT || process.env.E2E_CLUSTER_FINGERPRINT+ const expectedImagesGeneratedAt = process.env.E2E_EXPECTED_IMAGES_GENERATED_AT+ validateGfsRuntimeImageMarker(markerData ?? {}, {+ imageSource: expectedImageSource,+ ...(expectedImageTag !== undefined ? { imageTag: expectedImageTag } : {}),+ ...(expectedClusterFingerprint ? { clusterFingerprint: expectedClusterFingerprint } : {}),+ ...(expectedImagesGeneratedAt ? { imagesGeneratedAt: expectedImagesGeneratedAt } : {}),+ })+}++async function kubectl(args: string[], timeout = 240_000): Promise<string> {+ const result = await execFileAsync('kubectl', ['--context', kubectlContext(), ...args], {+ encoding: 'utf8',+ timeout,+ maxBuffer: 2 * 1024 * 1024,+ })+ return result.stdout+}++async function renderedWriterFlag(): Promise<string> {+ return (+ await kubectl([+ '-n',+ GFS_NAMESPACE,+ 'get',+ `deployment/${GFS_WRITER_DEPLOYMENT}`,+ '-o',+ `jsonpath={range .spec.template.spec.containers[*]}{range .env[?(@.name=="${GFS_V2_WRITER_ENV}")]}{.value}{end}{end}`,+ ])+ ).trim()+}++async function waitForRenderedWriterFlag(expected: boolean): Promise<void> {+ const expectedValue = String(expected)+ const deadline = Date.now() + 240_000+ let lastValue = ''+ while (Date.now() < deadline) {+ lastValue = await renderedWriterFlag()+ if (lastValue === expectedValue) return+ await new Promise(resolve => setTimeout(resolve, 1_000))+ }+ throw new Error(+ `GFS Upload v2 runtime flag did not converge to ${expectedValue}; observed ${lastValue || '<empty>'}`+ )+}++async function waitForDeployment(deployment: string): Promise<void> {+ await kubectl([+ '-n',+ GFS_NAMESPACE,+ 'rollout',+ 'status',+ `deployment/${deployment}`,+ '--timeout=240s',+ ])+}++/**+ * Changes the HCC-owned source flag and waits for the generated writer to+ * converge. The caller must restore the original value in a finally block.+ */+export async function setGfsUploadV2Enabled(enabled: boolean): Promise<void> {+ await assertOwnedRuntimeContext()+ await kubectl([+ '-n',+ HCC_NAMESPACE,+ 'set',+ 'env',+ `deployment/${HCC_DEPLOYMENT}`,+ `${GFS_V2_HCC_ENV}=${String(enabled)}`,+ ])+ await kubectl([+ '-n',+ HCC_NAMESPACE,+ 'rollout',+ 'status',+ `deployment/${HCC_DEPLOYMENT}`,+ '--timeout=240s',+ ])+ await waitForRenderedWriterFlag(enabled)+ await waitForDeployment(GFS_WRITER_DEPLOYMENT)+ await waitForDeployment(GFS_READER_DEPLOYMENT)+}++export async function readGfsUploadV2Enabled(): Promise<boolean> {+ const value = await renderedWriterFlag()+ if (value !== 'true' && value !== 'false') {+ throw new Error(+ `GFS writer rendered an invalid ${GFS_V2_WRITER_ENV} value: ${value || '<empty>'}`+ )+ }+ return value === 'true'+}++export async function restartGfsWriter(): Promise<void> {+ await assertOwnedRuntimeContext()+ // Restart the real writer deployment while the upload is visibly active.+ // A rollout gives the relay a bounded termination path; deleting the pod+ // directly can leave an already-streaming PUT socket open until the full+ // five-minute part timeout, turning this journey into a transport-timeout+ // test instead of a restart/recovery test.+ await kubectl(['-n', GFS_NAMESPACE, 'rollout', 'restart', `deployment/${GFS_WRITER_DEPLOYMENT}`])+ await waitForDeployment(GFS_WRITER_DEPLOYMENT)+}++/** Revoke only the exact seeded user grant used by the Desktop E2E. */+export async function revokeGfsUserWriteGrant(resourceId: string, userId: string): Promise<void> {+ await assertOwnedRuntimeContext()+ runControlPostgresSql(`+ DELETE FROM gfs_grants+ WHERE drive = 'main'+ AND resource_id = ${sqlLiteral(resourceId)}::uuid+ AND subject_type = 'user'+ AND subject_id = ${sqlLiteral(userId)};+ `)+}++/**+ * Legacy fallback must not create an indexed Upload v2 session. Scope the+ * assertion to the exact seeded parent/name/owner tuple so unrelated active+ * sessions cannot make the negative journey flaky.+ */+export function countGfsCreateSessions(+ parentResourceId: string,+ resourceName: string,+ ownerSubject: string+): number {+ const value = firstDataLine(+ runControlPostgresSql(`+ SELECT COUNT(*)::int+ FROM gfs_upload_sessions+ WHERE drive = 'main'+ AND owner_subject = ${sqlLiteral(ownerSubject)}+ AND operation = 'create'+ AND parent_rid = ${sqlLiteral(parentResourceId)}+ AND resource_name = ${sqlLiteral(resourceName)};+ `)+ )+ const count = Number(value)+ if (!Number.isSafeInteger(count) || count < 0) {+ throw new Error(`invalid GFS v2 session count: ${value || '<empty>'}`)+ }+ return count+}diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex 0b79a7bd0..7f2e6a657 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.63",+ "version": "0.1.64","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.63",+ "version": "0.1.64","devDependencies": {"@types/jsonwebtoken": "^9.0.10","@types/node": "^22.0.0",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex cdb0121e7..3912175de 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.63",+ "version": "0.1.64","private": true,"type": "module","scripts": {