Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(gfs): close upload review gaps and runtime gates

Alfredo Lopez893011ea8226
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index e795575b3..0d9088665 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -428,6 +428,9 @@ jobs:
- name: Test minikube GFS Upload v2 profile contract
run: bash scripts/tests/test-minikube-gfs-upload-v2-profile.sh
+ - name: Test GFS Upload v2 cross-client geometry contract
+ run: bash scripts/tests/test-gfs-upload-v2-geometry-contract.sh
+
# The NetworkPolicy probe runs on an E2E-only fixture that the default
# (pulled) setup never acquires. This proves the target says so up front
# instead of timing out on ErrImageNeverPull, and that it still builds
diff --git a/control-api/src/app.ts b/control-api/src/app.ts
index 67aa412c0..fff8ffbde 100644
--- a/control-api/src/app.ts
+++ b/control-api/src/app.ts
@@ -63,7 +63,7 @@ const RPC_HOST_ACCESS_PATH = /^\/api\/v1\/rpc\/access\/users\/[^/]+\/mcp-hosts\/
// proxy untouched; parsing them as JSON would either reject the first binary
// byte or buffer the whole part in the control plane.
const GFS_UPLOAD_PART_PATH =
- /^\/api\/v1\/(?:gfs\/proxy\/v1|external\/gfs)\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i
+ /^\/api\/v1\/(?:gfs\/proxy\/v1|external\/gfs)\/uploads\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/parts\/[0-9]+$/i
export function createApp(gateway: K8sGateway) {
const traceIngestDb: DbClient = meterTracingDbClient({
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.ts
index d5fa6c032..530f87552 100644
--- a/control-api/src/middleware/gfsUploadAdmission.ts
+++ b/control-api/src/middleware/gfsUploadAdmission.ts
@@ -65,7 +65,14 @@ export function createObservedGfsUploadPartBody(
}
function sourceIp(req: ExternalAuthedRequest): string {
- const forwarded = String(req.headers['x-gfs-upload-source-ip'] || '').trim()
+ // Only the authenticated external-rest-api service may assert the original
+ // edge address. Direct callers of Control API (including other valid
+ // service identities) must use the socket/proxy-derived address so they
+ // cannot rotate an arbitrary forwarded header to evade the IP bucket.
+ const forwarded =
+ req.internalService?.name === 'external-rest-api'
+ ? String(req.headers['x-gfs-upload-source-ip'] || '').trim()
+ : ''
if (forwarded && forwarded.length <= 64 && isIP(forwarded)) return forwarded
const direct = String(req.ip || req.socket.remoteAddress || 'unknown').trim()
return direct.length <= 64 ? direct : 'unknown'
@@ -132,7 +139,7 @@ export function gfsUploadAdmission(
if (isPart) {
const parsed = declaredGfsUploadPartBytes(req)
if (!parsed.ok) {
- gfsUploadAdmissionBytesTotal.inc({ result: 'rejected' }, 0)
+ gfsUploadAdmissionRequestsTotal.inc({ limit: 'part_body', result: 'rejected' }, 1)
res.status(parsed.status).json({ error: parsed.error })
return
}
diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.ts
index adc573f95..45e2f00ec 100644
--- a/control-api/src/routes/gfs/proxy.ts
+++ b/control-api/src/routes/gfs/proxy.ts
@@ -29,7 +29,8 @@ const PASSTHROUGH_RESPONSE_HEADERS = [
'upload-checksum',
'retry-after',
]
-const UPLOAD_PART_PATH = /^\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+(?:\?|$)/i
+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+const UPLOAD_PART_PATH = new RegExp(`^/v1/uploads/${UUID_PATH}/parts/[0-9]+(?:\\?|$)`, 'i')
const UPLOAD_PATH = /^\/v1\/(?:capabilities|uploads)(?:\/|\?|$)/i
const STREAM_REQUEST_HEADERS = [
'content-type',
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index d5d8ce30c..f7e44cf3d 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -139,11 +139,29 @@ export type RateLimitConcurrencyLease = {
release: () => Promise<void>
}
+export type RateLimitConcurrencyLeaseOptions = {
+ /** Test/integration seam; production callers use the dedicated pool client. */
+ client?: PoolClient
+}
+
let concurrencyClient: PoolClient | null = null
let concurrencyClientPromise: Promise<PoolClient> | null = null
-const heldConcurrencySlots = new Set<string>()
+// Session-scoped advisory locks are re-entrant only on the same PostgreSQL
+// connection. Keep the local guard per client so an integration test (and any
+// future multi-pool caller) still exercises the database lock across sessions.
+const heldConcurrencySlotsByClient = new WeakMap<object, Set<string>>()
let concurrencyOperation = Promise.resolve()
+function heldSlotsFor(client: PoolClient): Set<string> {
+ const key = client as unknown as object
+ let slots = heldConcurrencySlotsByClient.get(key)
+ if (!slots) {
+ slots = new Set<string>()
+ heldConcurrencySlotsByClient.set(key, slots)
+ }
+ return slots
+}
+
async function serializeConcurrencyOperation<T>(operation: () => Promise<T>): Promise<T> {
const previous = concurrencyOperation
let finish!: () => void
@@ -158,8 +176,14 @@ async function serializeConcurrencyOperation<T>(operation: () => Promise<T>): Pr
}
}
-function advisoryKey(value: string): number {
- return createHash('sha256').update(value).digest().readInt32BE(0)
+type AdvisoryLockKey = {
+ high: number
+ low: number
+}
+
+function advisoryKey(value: string): AdvisoryLockKey {
+ const digest = createHash('sha256').update(value).digest()
+ return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }
}
async function getConcurrencyClient(): Promise<PoolClient> {
@@ -173,7 +197,7 @@ async function getConcurrencyClient(): Promise<PoolClient> {
'rate limiter concurrency connection failed'
)
if (concurrencyClient === client) concurrencyClient = null
- heldConcurrencySlots.clear()
+ heldSlotsFor(client).clear()
client.release(true)
})
return client
@@ -188,16 +212,21 @@ async function getConcurrencyClient(): Promise<PoolClient> {
async function unlockSlots(
client: PoolClient,
- slots: Array<{ key: number; slot: number; identity: string }>
-): Promise<void> {
+ slots: Array<{ key: AdvisoryLockKey; identity: string }>
+): Promise<boolean> {
+ const heldSlots = heldSlotsFor(client)
+ let success = true
for (const acquired of slots.reverse()) {
- heldConcurrencySlots.delete(acquired.identity)
try {
await client.query('SELECT pg_advisory_unlock($1::integer, $2::integer)', [
- acquired.key,
- acquired.slot,
+ acquired.key.high,
+ acquired.key.low,
])
+ heldSlots.delete(acquired.identity)
} catch (error) {
+ // The lock state is unknown after an unlock error. Keep the local guard
+ // until the connection error path invalidates this client; otherwise a
+ // same-session reentrant pg_try_advisory_lock could admit a duplicate.
rootLogger.warn(
{
event: 'rate_limit_concurrency_unlock_error',
@@ -205,8 +234,20 @@ async function unlockSlots(
},
'rate limiter concurrency unlock failed'
)
+ success = false
+ break
}
}
+ return success
+}
+
+async function invalidateDedicatedConcurrencyClient(client: PoolClient): Promise<void> {
+ if (concurrencyClient !== client) return
+ concurrencyClient = null
+ heldSlotsFor(client).clear()
+ // Advisory locks are session-scoped. Destroying this dedicated client is the
+ // only fail-closed way to release a lock whose unlock response is unknown.
+ client.release(true)
}
/**
@@ -216,7 +257,8 @@ async function unlockSlots(
* reentrancy from admitting the same slot twice.
*/
export async function acquireRateLimitConcurrencyLease(
- requirements: readonly RateLimitConcurrencyRequirement[]
+ requirements: readonly RateLimitConcurrencyRequirement[],
+ options: RateLimitConcurrencyLeaseOptions = {}
): Promise<RateLimitConcurrencyLease> {
for (const requirement of requirements) {
if (!Number.isSafeInteger(requirement.maxConcurrent) || requirement.maxConcurrent < 1)
@@ -225,7 +267,7 @@ export async function acquireRateLimitConcurrencyLease(
return serializeConcurrencyOperation(async () => {
let client: PoolClient
try {
- client = await getConcurrencyClient()
+ client = options.client ?? (await getConcurrencyClient())
} catch (error) {
rootLogger.warn(
{
@@ -237,32 +279,39 @@ export async function acquireRateLimitConcurrencyLease(
return { allowed: false, backendAvailable: false, release: async () => undefined }
}
- const acquired: Array<{ key: number; slot: number; identity: string }> = []
+ const acquired: Array<{ key: AdvisoryLockKey; identity: string }> = []
try {
for (const requirement of requirements) {
- const key = advisoryKey(requirement.bucketKey)
- let selected: { key: number; slot: number; identity: string } | null = null
+ const bucketKey = advisoryKey(requirement.bucketKey)
+ let selected: { key: AdvisoryLockKey; identity: string } | null = null
for (let slot = 0; slot < requirement.maxConcurrent; slot += 1) {
- const identity = `${key}:${slot}`
- if (heldConcurrencySlots.has(identity)) continue
+ // Keep both halves of the SHA-256-derived key in the PostgreSQL
+ // advisory lock identity. The slot is folded into the low half so
+ // distinct buckets do not collide merely because their first 32
+ // digest bits match.
+ const key = { high: bucketKey.high, low: bucketKey.low ^ slot }
+ const identity = `${key.high}:${key.low}`
+ if (heldSlotsFor(client).has(identity)) continue
const result = await client.query<{ acquired: boolean }>(
'SELECT pg_try_advisory_lock($1::integer, $2::integer) AS acquired',
- [key, slot]
+ [key.high, key.low]
)
if (result.rows[0]?.acquired === true) {
- selected = { key, slot, identity }
- heldConcurrencySlots.add(identity)
+ selected = { key, identity }
+ heldSlotsFor(client).add(identity)
acquired.push(selected)
break
}
}
if (!selected) {
- await unlockSlots(client, acquired)
+ const unlocked = await unlockSlots(client, acquired)
+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)
return { allowed: false, backendAvailable: true, release: async () => undefined }
}
}
} catch (error) {
- await unlockSlots(client, acquired)
+ const unlocked = await unlockSlots(client, acquired)
+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)
rootLogger.warn(
{
event: 'rate_limit_concurrency_db_error',
@@ -280,7 +329,10 @@ export async function acquireRateLimitConcurrencyLease(
release: async () => {
if (released) return
released = true
- await serializeConcurrencyOperation(() => unlockSlots(client, acquired))
+ await serializeConcurrencyOperation(async () => {
+ const unlocked = await unlockSlots(client, acquired)
+ if (!unlocked) await invalidateDedicatedConcurrencyClient(client)
+ })
},
}
})
diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.ts
index b8b5e5375..93cd1c50c 100644
--- a/control-api/test/gfsUploadAdmission.test.ts
+++ b/control-api/test/gfsUploadAdmission.test.ts
@@ -7,12 +7,26 @@ import {
declaredGfsUploadPartBytes,
gfsUploadAdmission,
} from '../src/middleware/gfsUploadAdmission.js'
+import {
+ requireInternalService,
+ requireInternalToken,
+} from '../src/middleware/internalServiceAuth.js'
-const { acquireMock, checkMock, metricDec, metricInc, releaseMock } = vi.hoisted(() => ({
+const {
+ acquireMock,
+ checkMock,
+ metricActiveInc,
+ metricBytesInc,
+ metricDec,
+ metricRequestsInc,
+ releaseMock,
+} = vi.hoisted(() => ({
acquireMock: vi.fn(),
checkMock: vi.fn(),
+ metricActiveInc: vi.fn(),
+ metricBytesInc: vi.fn(),
metricDec: vi.fn(),
- metricInc: vi.fn(),
+ metricRequestsInc: vi.fn(),
releaseMock: vi.fn(),
}))
@@ -27,6 +41,10 @@ vi.mock('../src/config.js', () => ({
gfsUploadMaxActivePerIp: 16,
gfsUploadMaxActiveGlobal: 32,
gfsUploadMaxPartBytes: 16 * 1024 * 1024,
+ internalServiceTokens: {
+ 'external-rest-api': 'dev-external-rest-api-token',
+ 'rpc-proxy': 'dev-rpc-proxy-token',
+ },
},
}))
@@ -36,9 +54,9 @@ vi.mock('../src/services/rateLimiterService.js', () => ({
}))
vi.mock('../src/observability/metrics.js', () => ({
- gfsUploadAdmissionActiveRequests: { inc: metricInc, dec: metricDec },
- gfsUploadAdmissionBytesTotal: { inc: metricInc },
- gfsUploadAdmissionRequestsTotal: { inc: metricInc },
+ gfsUploadAdmissionActiveRequests: { inc: metricActiveInc, dec: metricDec },
+ gfsUploadAdmissionBytesTotal: { inc: metricBytesInc },
+ gfsUploadAdmissionRequestsTotal: { inc: metricRequestsInc },
}))
function rateCheck(
@@ -62,7 +80,7 @@ function rateCheck(
}
}
-function buildPartApp() {
+function buildPartApp(serviceName?: string) {
const app = express()
app.put(
'/external/gfs/uploads/:id/parts/:part',
@@ -70,6 +88,7 @@ function buildPartApp() {
;(req as typeof req & { externalAuth: { userId: string } }).externalAuth = {
userId: 'user-1',
}
+ if (serviceName) req.internalService = { name: serviceName }
next()
},
gfsUploadAdmission,
@@ -97,11 +116,34 @@ function buildLifecycleApp() {
return app
}
+function buildAuthenticatedPartApp() {
+ const app = express()
+ app.put(
+ '/external/gfs/uploads/:id/parts/:part',
+ requireInternalToken,
+ requireInternalService('external-rest-api'),
+ (req, _res, next) => {
+ ;(req as typeof req & { externalAuth: { userId: string } }).externalAuth = {
+ userId: 'user-1',
+ }
+ next()
+ },
+ gfsUploadAdmission,
+ (req, res) => {
+ req.resume()
+ req.once('end', () => res.status(204).end())
+ }
+ )
+ return app
+}
+
beforeEach(() => {
acquireMock.mockReset()
checkMock.mockReset()
+ metricActiveInc.mockReset()
+ metricBytesInc.mockReset()
metricDec.mockReset()
- metricInc.mockReset()
+ metricRequestsInc.mockReset()
releaseMock.mockReset()
checkMock.mockResolvedValue(rateCheck())
acquireMock.mockResolvedValue({
@@ -128,6 +170,21 @@ describe('GFS upload admission', () => {
expect(acquireMock).not.toHaveBeenCalled()
})
+ it('records a rejected part-body request without charging the byte counter', async () => {
+ const response = await request(buildPartApp())
+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')
+ .set('content-type', 'application/offset+octet-stream')
+ .set('content-length', '4')
+ .set('upload-chunk-length', '3')
+ .send(Buffer.from('abc'))
+
+ expect(response.status).toBe(400)
+ expect(metricRequestsInc).toHaveBeenCalledWith({ limit: 'part_body', result: 'rejected' }, 1)
+ expect(metricRequestsInc).toHaveBeenCalledTimes(1)
+ expect(metricBytesInc).not.toHaveBeenCalled()
+ expect(metricActiveInc).not.toHaveBeenCalled()
+ })
+
it('charges part bytes in fixed 1 MiB units before forwarding and releases its active slot', async () => {
const bytes = Buffer.alloc(1024 * 1024 + 1, 0x61)
const response = await request(buildPartApp())
@@ -154,6 +211,55 @@ describe('GFS upload admission', () => {
expect(releaseMock).toHaveBeenCalledTimes(1)
})
+ it('ignores a spoofed source IP unless the authenticated external relay asserted it', async () => {
+ const bytes = Buffer.from('a')
+ await request(buildPartApp())
+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')
+ .set('content-type', 'application/offset+octet-stream')
+ .set('upload-chunk-length', String(bytes.length))
+ .set('x-gfs-upload-source-ip', '203.0.113.7')
+ .send(bytes)
+
+ expect(checkMock.mock.calls[1][0]).not.toBe('gfs-upload:req:ip:203.0.113.7')
+ checkMock.mockClear()
+
+ await request(buildPartApp('external-rest-api'))
+ .put('/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0')
+ .set('content-type', 'application/offset+octet-stream')
+ .set('upload-chunk-length', String(bytes.length))
+ .set('x-gfs-upload-source-ip', '203.0.113.7')
+ .send(bytes)
+
+ expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')
+ })
+
+ it('binds source-IP assertion to the real internal service token wiring', async () => {
+ const app = buildAuthenticatedPartApp()
+ const path = '/external/gfs/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0'
+ const headers = {
+ 'content-type': 'application/offset+octet-stream',
+ 'upload-chunk-length': '1',
+ 'x-gfs-upload-source-ip': '203.0.113.7',
+ }
+ await request(app)
+ .put(path)
+ .auth('dev-rpc-proxy-token', { type: 'bearer' })
+ .set('x-service-token', 'rpc-proxy')
+ .set(headers)
+ .send(Buffer.from('a'))
+ .expect(401)
+
+ checkMock.mockClear()
+ await request(app)
+ .put(path)
+ .auth('dev-external-rest-api-token', { type: 'bearer' })
+ .set('x-service-token', 'external-rest-api')
+ .set(headers)
+ .send(Buffer.from('a'))
+ .expect(204)
+ expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')
+ })
+
it('returns the same 429 contract when the active-request budget is exhausted', async () => {
acquireMock.mockResolvedValueOnce({
allowed: false,
diff --git a/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts
new file mode 100644
index 000000000..5e793c6b1
--- /dev/null
+++ b/control-api/test/gfsUploadFinalizer.realPostgres.integration.test.ts
@@ -0,0 +1,245 @@
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { createHash, randomBytes, randomUUID } from 'node:crypto'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Pool } from 'pg'
+import { DbAuditSink } from '../../gfs-controller/src/authz/permissionClient.js'
+import type { PermissionClient } from '../../gfs-controller/src/authz/permissionClient.js'
+import { PgBlobStagingStore } from '../../gfs-controller/src/db/blobStaging.js'
+import { GfsWriteService, PgTransactor } from '../../gfs-controller/src/db/writeStore.js'
+import { BlobStore } from '../../gfs-controller/src/storage/blobStore.js'
+import { createGfsUploadFinalizer } from '../../gfs-controller/src/upload/uploadFinalizer.js'
+import type {
+ UploadPartRow,
+ UploadSessionRow,
+} from '../../gfs-controller/src/upload/uploadSession.js'
+import { initDb } from '../src/db.js'
+
+const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
+const describeRealPostgres = adminUrl ? describe : describe.skip
+const BODY = Buffer.alloc(1024 * 1024, 0x42)
+const BODY_SHA256 = createHash('sha256').update(BODY).digest('hex')
+const HOST_SUBJECT = 'host:1st:gfs-test/upload-finalizer'
+
+function databaseUrl(baseUrl: string, database: string): string {
+ const url = new URL(baseUrl)
+ url.pathname = `/${database}`
+ return url.toString()
+}
+
+function quoteIdent(value: string): string {
+ return `"${value.replace(/"/g, '""')}"`
+}
+
+function session(uploadId: string, drive: string, parentRid: string): UploadSessionRow {
+ return {
+ uploadId,
+ idempotencyKey: randomUUID(),
+ drive,
+ ownerSubject: HOST_SUBJECT,
+ primarySubject: HOST_SUBJECT,
+ operation: 'create',
+ requestFingerprint: 'real-finalizer-test',
+ parentRid,
+ resourceRid: null,
+ resourceName: 'payload.bin',
+ ifMatch: null,
+ expectedBytes: BODY.length,
+ partBytes: BODY.length,
+ partCount: 1,
+ wholeSha256: BODY_SHA256,
+ committedBytes: BODY.length,
+ contiguousBytes: BODY.length,
+ committedPartCount: 1,
+ activePartCount: 0,
+ sessionEpoch: 0,
+ state: 'finalizing',
+ resultResourceId: null,
+ resultVersion: null,
+ resultSha256: null,
+ failureCode: null,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+}
+
+function part(uploadId: string, root: string): UploadPartRow {
+ return {
+ uploadId,
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: BODY.length,
+ sha256: BODY_SHA256,
+ state: 'committed',
+ stagingPath: join(root, '.uploads', uploadId, 'parts', '0.part'),
+ leaseEpoch: 0,
+ }
+}
+
+function permissions(allowed: () => boolean): PermissionClient {
+ return {
+ permissionEpoch: () => ({ generation: 0, bypassed: false }),
+ authorize: async () => ({ allowed: allowed() }),
+ } as unknown as PermissionClient
+}
+
+describeRealPostgres('GFS upload finalizer on real PostgreSQL + BlobStore', () => {
+ const database = `gfs_upload_finalizer_${randomBytes(6).toString('hex')}`
+ const connectionString = databaseUrl(
+ adminUrl ?? 'postgresql://[email protected]/postgres',
+ database
+ )
+ const drive = `finalizer-${randomBytes(4).toString('hex')}`
+ let adminPool: Pool
+ let pool: Pool
+ let storageRoot: string
+ let blobRoot: string
+ let parentId: string
+
+ beforeAll(async () => {
+ adminPool = new Pool({ connectionString: adminUrl })
+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
+ pool = new Pool({ connectionString })
+ await initDb({ connect: () => pool.connect() })
+ storageRoot = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-parts-'))
+ blobRoot = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-blobs-'))
+ parentId = randomUUID()
+ await pool.query(
+ `INSERT INTO gfs_resources
+ (resource_id, drive, parent_resource_id, name, kind, path_cache, version, bytes)
+ VALUES ($1, $2, NULL, '', 'directory', '/', 0, 0)`,
+ [parentId, drive]
+ )
+ }, 60_000)
+
+ afterAll(async () => {
+ await pool?.query('DELETE FROM gfs_resources WHERE drive = $1', [drive]).catch(() => undefined)
+ await pool
+ ?.query('DELETE FROM gfs_upload_sessions WHERE drive = $1', [drive])
+ .catch(() => undefined)
+ await pool?.end()
+ await rm(storageRoot, { recursive: true, force: true }).catch(() => undefined)
+ await rm(blobRoot, { recursive: true, force: true }).catch(() => undefined)
+ if (!adminPool) return
+ await adminPool.query(
+ `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
+ WHERE datname = $1 AND pid <> pg_backend_pid()`,
+ [database]
+ )
+ await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
+ await adminPool.end()
+ })
+
+ async function seedUpload(): Promise<{ upload: UploadSessionRow; uploadPart: UploadPartRow }> {
+ const uploadId = randomUUID()
+ const upload = session(uploadId, drive, parentId)
+ const uploadPart = part(uploadId, storageRoot)
+ await mkdir(join(storageRoot, '.uploads', uploadId, 'parts'), { recursive: true })
+ await writeFile(uploadPart.stagingPath, BODY)
+ await pool.query(
+ `INSERT INTO gfs_upload_sessions
+ (upload_id, idempotency_key, drive, owner_subject, primary_subject, operation,
+ request_fingerprint, parent_rid, resource_name, expected_bytes, part_bytes,
+ part_count, whole_sha256, committed_bytes, contiguous_bytes,
+ committed_part_count, active_part_count, session_epoch, state, expires_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)`,
+ [
+ upload.uploadId,
+ upload.idempotencyKey,
+ upload.drive,
+ upload.ownerSubject,
+ upload.primarySubject,
+ upload.operation,
+ upload.requestFingerprint,
+ upload.parentRid,
+ upload.resourceName,
+ upload.expectedBytes,
+ upload.partBytes,
+ upload.partCount,
+ upload.wholeSha256,
+ upload.committedBytes,
+ upload.contiguousBytes,
+ upload.committedPartCount,
+ upload.activePartCount,
+ upload.sessionEpoch,
+ upload.state,
+ upload.expiresAt,
+ ]
+ )
+ await pool.query(
+ `INSERT INTO gfs_upload_parts
+ (upload_id, part_number, offset_bytes, length_bytes, sha256, state,
+ staging_path, lease_epoch, committed_at)
+ VALUES ($1, 0, 0, $2, $3, 'committed', $4, 0, now())`,
+ [upload.uploadId, BODY.length, BODY_SHA256, uploadPart.stagingPath]
+ )
+ return { upload, uploadPart }
+ }
+
+ function finalizer(allow: () => boolean): ReturnType<typeof createGfsUploadFinalizer> {
+ const blobs = new BlobStore(blobRoot, 'writer')
+ const writes = new GfsWriteService(new PgTransactor(pool), blobs, new PgBlobStagingStore(pool))
+ return createGfsUploadFinalizer({
+ pool,
+ storageMountPath: storageRoot,
+ permissions: permissions(allow),
+ writeService: writes,
+ audit: new DbAuditSink(pool),
+ })
+ }
+
+ it('publishes the resource and receipt atomically through the real writer', async () => {
+ const { upload, uploadPart } = await seedUpload()
+ const result = await finalizer(() => true)(upload, [uploadPart])
+ expect(result.sha256).toBe(BODY_SHA256)
+ const receipt = await pool.query(
+ `SELECT state, result_resource_id, result_version, result_sha256
+ FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [upload.uploadId]
+ )
+ expect(receipt.rows[0]).toMatchObject({
+ state: 'completed',
+ result_resource_id: result.resourceId,
+ result_version: result.version,
+ result_sha256: BODY_SHA256,
+ })
+ const resource = await pool.query(
+ `SELECT drive, parent_resource_id, bytes, content_sha256
+ FROM gfs_resources WHERE resource_id = $1`,
+ [result.resourceId]
+ )
+ expect(resource.rows[0]).toMatchObject({
+ drive,
+ parent_resource_id: parentId,
+ bytes: BODY.length,
+ content_sha256: BODY_SHA256,
+ })
+ const manifests = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,
+ [result.resourceId]
+ )
+ expect(manifests.rows[0]?.count).toBe(0)
+ }, 30_000)
+
+ it('rolls back the real resource and receipt when authorization is denied', async () => {
+ const { upload, uploadPart } = await seedUpload()
+ await expect(finalizer(() => false)(upload, [uploadPart])).rejects.toMatchObject({
+ code: 'forbidden',
+ })
+ const receipt = await pool.query(
+ `SELECT state, result_resource_id FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [upload.uploadId]
+ )
+ expect(receipt.rows[0]).toEqual({ state: 'finalizing', result_resource_id: null })
+ const resource = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_resources WHERE resource_id = $1`,
+ [upload.uploadId]
+ )
+ expect(resource.rows[0]?.count).toBe(0)
+ const manifests = await pool.query(
+ `SELECT count(*)::int AS count FROM gfs_blob_manifests WHERE resource_id = $1`,
+ [upload.uploadId]
+ )
+ expect(manifests.rows[0]?.count).toBe(0)
+ }, 30_000)
+})
diff --git a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts
index 2879879ce..4d086c91f 100644
--- a/control-api/test/services.rateLimiter.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiter.realPostgres.integration.test.ts
@@ -2,7 +2,10 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import { initDb } from '../src/db.js'
-import { checkAndIncrementWithQuery } from '../src/services/rateLimiterService.js'
+import {
+ acquireRateLimitConcurrencyLease,
+ checkAndIncrementWithQuery,
+} from '../src/services/rateLimiterService.js'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
@@ -69,29 +72,82 @@ describeRealPostgres('rate limiter atomicity on real PostgreSQL', () => {
expect(Number(persisted.rows[0]?.count)).toBe(32)
})
- it('serializes a bounded advisory slot across independent PostgreSQL sessions', async () => {
+ it('serializes the production bounded advisory lease across independent PostgreSQL sessions', async () => {
const key = `gfs-upload-slot:${randomBytes(8).toString('hex')}`
const first = await pool.connect()
const second = await pool.connect()
try {
- const acquiredFirst = await first.query<{ acquired: boolean }>(
- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
- [key]
+ const firstLease = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: key, maxConcurrent: 1 }],
+ { client: first }
)
- const acquiredSecond = await second.query<{ acquired: boolean }>(
- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
- [key]
+ expect(firstLease).toMatchObject({ allowed: true, backendAvailable: true })
+
+ const secondLease = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: key, maxConcurrent: 1 }],
+ { client: second }
+ )
+ expect(secondLease).toMatchObject({ allowed: false, backendAvailable: true })
+
+ await firstLease.release()
+ const acquiredAfterRelease = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: key, maxConcurrent: 1 }],
+ { client: second }
+ )
+ expect(acquiredAfterRelease).toMatchObject({ allowed: true, backendAvailable: true })
+ await acquiredAfterRelease.release()
+ } finally {
+ first.release()
+ second.release()
+ }
+ })
+
+ it('uses distinct slots for capacity two and rolls back partial multi-bucket acquisition', async () => {
+ const slotKey = `gfs-upload-slots:${randomBytes(8).toString('hex')}`
+ const rollbackKey = `gfs-upload-rollback:${randomBytes(8).toString('hex')}`
+ const freeKey = `gfs-upload-free:${randomBytes(8).toString('hex')}`
+ const first = await pool.connect()
+ const second = await pool.connect()
+ try {
+ const firstSlot = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: slotKey, maxConcurrent: 2 }],
+ { client: first }
+ )
+ const secondSlot = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: slotKey, maxConcurrent: 2 }],
+ { client: second }
)
- expect(acquiredFirst.rows[0]?.acquired).toBe(true)
- expect(acquiredSecond.rows[0]?.acquired).toBe(false)
+ expect(firstSlot.allowed).toBe(true)
+ expect(secondSlot.allowed).toBe(true)
+ const exhausted = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: slotKey, maxConcurrent: 2 }],
+ { client: first }
+ )
+ expect(exhausted.allowed).toBe(false)
- await first.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])
- const acquiredAfterRelease = await second.query<{ acquired: boolean }>(
- `SELECT pg_try_advisory_lock(hashtext($1), 0) AS acquired`,
- [key]
+ const held = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: rollbackKey, maxConcurrent: 1 }],
+ { client: first }
+ )
+ expect(held.allowed).toBe(true)
+ const partial = await acquireRateLimitConcurrencyLease(
+ [
+ { bucketKey: freeKey, maxConcurrent: 1 },
+ { bucketKey: rollbackKey, maxConcurrent: 1 },
+ ],
+ { client: second }
)
- expect(acquiredAfterRelease.rows[0]?.acquired).toBe(true)
- await second.query(`SELECT pg_advisory_unlock(hashtext($1), 0)`, [key])
+ expect(partial.allowed).toBe(false)
+ const afterRollback = await acquireRateLimitConcurrencyLease(
+ [{ bucketKey: freeKey, maxConcurrent: 1 }],
+ { client: second }
+ )
+ expect(afterRollback.allowed).toBe(true)
+
+ await firstSlot.release()
+ await secondSlot.release()
+ await held.release()
+ await afterRollback.release()
} finally {
first.release()
second.release()
diff --git a/control-api/test/services.rateLimiter.test.ts b/control-api/test/services.rateLimiter.test.ts
index a3152aeeb..312782d4a 100644
--- a/control-api/test/services.rateLimiter.test.ts
+++ b/control-api/test/services.rateLimiter.test.ts
@@ -156,6 +156,26 @@ describe('rateLimiterService', () => {
expect(mockPoolConnect).toHaveBeenCalledTimes(1)
})
+ it('destroys the dedicated connection when an advisory unlock outcome is unknown', async () => {
+ const lease = await acquireRateLimitConcurrencyLease([
+ { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
+ ])
+ expect(lease.allowed).toBe(true)
+ mockClientQuery.mockRejectedValueOnce(new Error('connection lost during unlock'))
+
+ await lease.release()
+
+ expect(mockConcurrencyClient.release).toHaveBeenCalledWith(true)
+ // The fake client does not emulate PostgreSQL's session teardown; model
+ // the advisory lock release that the destroyed real connection provides.
+ advisoryLocks.clear()
+ const replacement = await acquireRateLimitConcurrencyLease([
+ { bucketKey: 'gfs-upload:unlock-failure', maxConcurrent: 1 },
+ ])
+ expect(replacement).toMatchObject({ allowed: true, backendAvailable: true })
+ await replacement.release()
+ })
+
it('fails open when pool.query throws (DB error)', async () => {
mockPoolQuery.mockRejectedValueOnce(new Error('connection refused'))
const r = await checkAndIncrement('test:bucket:failopen', 5)
diff --git a/control-ui/app/control-api/[...path]/route.ts b/control-ui/app/control-api/[...path]/route.ts
index 27bb44bf7..e8e6dba9d 100644
--- a/control-ui/app/control-api/[...path]/route.ts
+++ b/control-ui/app/control-api/[...path]/route.ts
@@ -134,7 +134,9 @@ function buildUpstreamUrl(req: NextRequest, path: string[]): string {
function isGfsUploadRoute(req: NextRequest, path: string[]): boolean {
if (req.method !== 'PUT') return false
const pathname = `/${path.join('/')}`
- return /^\/api\/v1\/gfs\/proxy\/v1\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i.test(pathname)
+ return /^\/api\/v1\/gfs\/proxy\/v1\/uploads\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/parts\/[0-9]+$/i.test(
+ pathname
+ )
}
function copyRequestHeaders(req: NextRequest): Headers {
diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsx
index 57d3f5e14..7b6174fb7 100644
--- a/control-ui/components/__tests__/GfsBrowser.test.tsx
+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx
@@ -384,6 +384,10 @@ describe('GfsBrowser', () => {
// Simulate an out-of-order in-flight part reporting a later absolute
// offset before the accumulator commits the lower-numbered part.
uploadInput?.onProgress?.({ uploadedBytes: 3 })
+ // A failed retry clears that part's in-flight contribution. The UI
+ // intentionally permits the truthful downward correction; the
+ // monotonic guard applies only to stale uploading snapshots.
+ uploadInput?.onProgress?.({ uploadedBytes: 1 })
uploadInput?.onState?.({
state: 'uploading',
session: { ...completed, state: 'uploading' },
@@ -441,10 +445,10 @@ describe('GfsBrowser', () => {
const progress = await within(uploadDialog).findByRole('progressbar', {
name: /Upload progress/,
})
- expect(progress).toHaveAttribute('value', '3')
+ expect(progress).toHaveAttribute('value', '2')
expect(progress).toHaveAttribute('aria-valuemin', '0')
expect(progress).toHaveAttribute('aria-valuemax', '4')
- expect(progress).toHaveAttribute('aria-valuenow', '3')
+ expect(progress).toHaveAttribute('aria-valuenow', '2')
fireEvent.click(within(uploadDialog).getByRole('button', { name: 'Pause' }))
await waitFor(() => expect(job.pause).toHaveBeenCalledTimes(1))
expect(await within(uploadDialog).findByRole('button', { name: 'Resume' })).toBeTruthy()
diff --git a/control-ui/e2e/gfs-upload-v2.spec.ts b/control-ui/e2e/gfs-upload-v2.spec.ts
index be0c0d86b..ac665c15d 100644
--- a/control-ui/e2e/gfs-upload-v2.spec.ts
+++ b/control-ui/e2e/gfs-upload-v2.spec.ts
@@ -11,8 +11,14 @@ import {
import {
GFS_UPLOAD_V2_BOUNDARIES,
createDiskUploadFixture,
+ createOversizedDiskUploadFixture,
removeDiskUploadFixture,
} from '../../tests/e2e/gfsUploadV2Fixtures'
+import {
+ readGfsUploadV2Enabled,
+ restartGfsWriter,
+ setGfsUploadV2Enabled,
+} from '../../tests/e2e/gfsUploadV2Runtime'
import { loginControlUi, openGlobalFileSystemFromSidebar } from './support/gfs-control-ui-session'
test.describe.configure({ mode: 'serial' })
@@ -211,3 +217,155 @@ test.describe('GFS Upload v2 — Control UI large-upload project', () => {
})
}
})
+
+test.describe('GFS Upload v2 — approved negative Control UI journeys', () => {
+ test.skip(
+ process.env.GFS_UPLOAD_V2_NEGATIVE_E2E !== '1',
+ 'Set GFS_UPLOAD_V2_NEGATIVE_E2E=1 only on an owned non-production dev host with the profile-scoped random URLs.'
+ )
+ test.setTimeout(45 * 60_000)
+
+ test('rejects a 209715201-byte file through the visible upload modal', async ({ page }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-oversize-ui')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ const source = await createOversizedDiskUploadFixture('.parquet', fixtureName)
+ try {
+ const current = await openWritableFolder(page, fixture.name)
+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()
+ const dialog = page.getByRole('dialog', { name: 'Upload file' })
+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)
+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()
+ await expect(
+ page.getByRole('status').filter({ hasText: 'GFS uploads are limited to 200 MB per file.' })
+ ).toBeVisible({ timeout: 15_000 })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 15_000 }
+ )
+ .toBeNull()
+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toHaveCount(
+ 0
+ )
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
+
+ test('falls back to the legacy path when v2 is disabled', async ({ page }) => {
+ const previous = await readGfsUploadV2Enabled()
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-legacy-ui')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ const source = await createDiskUploadFixture(2 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ await setGfsUploadV2Enabled(false)
+ const current = await openWritableFolder(page, fixture.name)
+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()
+ const dialog = page.getByRole('dialog', { name: 'Upload file' })
+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)
+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()
+ await expect(
+ page
+ .getByRole('status')
+ .filter({ hasText: 'Resumable upload is unavailable; using the legacy 16 MiB path.' })
+ ).toBeVisible({ timeout: 30_000 })
+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({
+ timeout: 120_000,
+ })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })
+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toBeVisible()
+ } finally {
+ try {
+ await setGfsUploadV2Enabled(previous)
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ }
+ })
+
+ test('recovers a visible upload after the writer deployment restarts', async ({ page }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-restart-ui')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ const current = await openWritableFolder(page, fixture.name)
+ const capabilitiesResponse = page.waitForResponse(
+ response =>
+ response.request().method() === 'GET' &&
+ response.url().includes('/gfs/proxy/v1/capabilities')
+ )
+ const createResponse = page.waitForResponse(
+ response =>
+ response.request().method() === 'POST' &&
+ response.url().includes('/gfs/proxy/v1/uploads') &&
+ !response.url().endsWith('/complete')
+ )
+ await page.getByRole('button', { name: 'Upload file', exact: true }).click()
+ const dialog = page.getByRole('dialog', { name: 'Upload file' })
+ await dialog.getByLabel('Choose file to upload').setInputFiles(source.filePath)
+ await dialog.getByRole('button', { name: 'Upload', exact: true }).click()
+ const capabilities = await capabilitiesResponse
+ const capabilitiesBody = (await capabilities.json()) as {
+ upload?: { resumableV2?: { enabled?: boolean } }
+ data?: { upload?: { resumableV2?: { enabled?: boolean } } }
+ }
+ const resumable =
+ capabilitiesBody.upload?.resumableV2 ?? capabilitiesBody.data?.upload?.resumableV2
+ expect(
+ capabilities.status(),
+ `${capabilities.url()} ${JSON.stringify(capabilitiesBody)}`
+ ).toBe(200)
+ expect(
+ resumable?.enabled,
+ 'restart journey must exercise the v2 writer, not legacy fallback'
+ ).toBe(true)
+ const created = await createResponse
+ const createdBody = await created.text()
+ expect(created.status(), `${created.url()} ${createdBody}`).toBeGreaterThanOrEqual(200)
+ expect(createdBody).toMatch(/uploadId|upload_id|data/)
+ const progress = page.getByRole('progressbar', {
+ name: `Upload progress for ${source.fileName}`,
+ })
+ // Establish the visible in-flight state, then inject a real writer pod
+ // loss. Completion after this point is the recovery assertion; the
+ // positive journeys separately enforce two intermediate progress values.
+ await expect(progress).toBeVisible({ timeout: 60_000 })
+ await restartGfsWriter()
+ await expect(page.getByText('File uploaded.', { exact: true })).toBeVisible({
+ timeout: 1_200_000,
+ })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })
+ await expect(current.getByRole('listitem').filter({ hasText: source.fileName })).toBeVisible()
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
+})
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index 11ce87cea..17f86ec52 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -495,6 +495,108 @@ describe('GfsUploadJob', () => {
expect(FakeXhr.requests).toHaveLength(2)
})
+ it('uses the extended bounded part budget while a writer is restarting', async () => {
+ const file = new File([new Uint8Array([23])], 'part-writer-restart.bin')
+ const session = {
+ uploadId: '14141414-1414-4141-8141-141414141414',
+ drive: 'main',
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ FakeXhr.statuses = [502, 502, 502, 502, 502, 204]
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status'))
+ return new Response(JSON.stringify({ ok: true, data: { session, parts: [] } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/complete'))
+ return new Response(
+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),
+ { status: 200 }
+ )
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-part-writer-restart' },
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(FakeXhr.requests).toHaveLength(6)
+ })
+
+ it('removes a failed part attempt from aggregate progress before retrying it', async () => {
+ const file = new File([new Uint8Array([31, 32, 33, 34])], 'progress-retry.bin')
+ const session = {
+ uploadId: '16161616-1616-4161-8161-161616161616',
+ drive: 'main',
+ expectedBytes: 4,
+ partBytes: 4,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ }
+ FakeXhr.statuses = [502, 204]
+ const progress: number[] = []
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
+ const method = init?.method ?? 'GET'
+ const url = String(_input)
+ if (method === 'GET' && url.endsWith('/capabilities'))
+ return new Response(JSON.stringify({ upload: { resumableV2: { enabled: true } } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/uploads'))
+ return new Response(JSON.stringify({ ok: true, data: session }), { status: 201 })
+ if (method === 'HEAD') return new Response(null, { status: 204 })
+ if (method === 'GET' && url.includes('/status'))
+ return new Response(JSON.stringify({ ok: true, data: { session, parts: [] } }), {
+ status: 200,
+ })
+ if (method === 'POST' && url.endsWith('/complete'))
+ return new Response(
+ JSON.stringify({ ok: true, data: { ...session, state: 'completed' } }),
+ { status: 200 }
+ )
+ throw new Error(`unexpected ${method} ${url}`)
+ })
+ )
+
+ await expect(
+ new GfsUploadJob({
+ file,
+ name: file.name,
+ target: { operation: 'create', parentRid: 'parent-progress-retry' },
+ onProgress: value => progress.push(value.uploadedBytes),
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+
+ expect(progress.some((value, index) => index > 0 && value < progress[index - 1]!)).toBe(true)
+ })
+
it('adopts a status-confirmed committed part after its XHR response is lost', async () => {
const file = new File([new Uint8Array([5, 6, 7, 8])], 'response-loss.bin', {
lastModified: 13,
diff --git a/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.ts b/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.ts
new file mode 100644
index 000000000..05783ff10
--- /dev/null
+++ b/control-ui/lib/__tests__/gfsUploadV2RuntimeMarker.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest'
+import { validateGfsRuntimeImageMarker } from '../../../tests/e2e/gfsUploadV2Runtime'
+
+const base = {
+ clusterFingerprint: 'cluster-a',
+ imagesGeneratedAt: '2026-08-12T18:00:00Z',
+}
+
+describe('GFS Upload v2 runtime image marker contract', () => {
+ it('accepts local image mode without a registry tag', () => {
+ expect(() =>
+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'local', imageTag: '' })
+ ).not.toThrow()
+ })
+
+ it('requires an immutable tag for GHCR image mode', () => {
+ expect(() =>
+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'ghcr', imageTag: '' })
+ ).toThrow(/GHCR image marker/)
+ })
+
+ it('rejects a marker that omits the imageTag field entirely', () => {
+ expect(() => validateGfsRuntimeImageMarker({ ...base, imageSource: 'local' })).toThrow(
+ /lacks cluster fingerprint/
+ )
+ })
+
+ it('rejects an image source outside the supported acquisition modes', () => {
+ expect(() =>
+ validateGfsRuntimeImageMarker({ ...base, imageSource: 'unknown', imageTag: '' })
+ ).toThrow(/unsupported imageSource/)
+ })
+
+ it('rejects a marker whose fingerprint or image acquisition stamp is stale', () => {
+ expect(() =>
+ validateGfsRuntimeImageMarker(
+ { ...base, imageSource: 'local', imageTag: '' },
+ { imageSource: 'local', imageTag: '', clusterFingerprint: 'cluster-b' }
+ )
+ ).toThrow(/clusterFingerprint/)
+ expect(() =>
+ validateGfsRuntimeImageMarker(
+ { ...base, imageSource: 'local', imageTag: '' },
+ { imagesGeneratedAt: '2026-08-12T19:00:00Z' }
+ )
+ ).toThrow(/imagesGeneratedAt/)
+ })
+})
diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.ts
index 9fd740f6e..536b54d86 100644
--- a/control-ui/lib/gfsFileUpload.ts
+++ b/control-ui/lib/gfsFileUpload.ts
@@ -14,6 +14,12 @@ const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3
const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100
export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
+/**
+ * A writer restart can keep the service endpoint unavailable for longer than
+ * the ordinary application-error retry budget. Keep this separate from the
+ * lifecycle budget so 502/503/504 recovery is longer but still bounded.
+ */
+export const GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS = 6
export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000
@@ -517,6 +523,13 @@ function ambiguousPartResponse(status: number | undefined): boolean {
return status === undefined || isAmbiguousUploadStatus(status)
}
+function partRetryAttempts(error: unknown): number {
+ const status = statusOf(error)
+ return status === undefined || status === 502 || status === 503 || status === 504
+ ? GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS
+ : GFS_UPLOAD_RETRY_MAX_ATTEMPTS
+}
+
function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {
return new Error(
`upload_part_outcome_unknown: could not prove whether part ${partNumber} committed; resume the persisted session`,
@@ -662,7 +675,8 @@ async function uploadPartWithRetries(
const blob = file.slice(part.offsetBytes, part.offsetBytes + part.lengthBytes)
const checksum = await sha256Part(blob)
let lastError: unknown
- for (let attempt = 0; attempt < 3; attempt += 1) {
+ let maxAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS
+ for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
try {
await putPart(
session.uploadId,
@@ -680,6 +694,7 @@ async function uploadPartWithRetries(
lastError = error
if (signal?.aborted) throw signal.reason ?? error
if (!assertRetryable(error)) throw error
+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(error))
onRetryableFailure?.()
if (ambiguousPartResponse(statusOf(error))) {
const reconciled = await reconcileAmbiguousPart(file, session, part, checksum, signal)
@@ -688,7 +703,7 @@ async function uploadPartWithRetries(
return
}
}
- if (attempt === 2) throw error
+ if (attempt + 1 >= maxAttempts) throw error
await delay(
retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
signal
@@ -707,6 +722,7 @@ async function runParts(
shouldPause: () => boolean,
onProgress?: (progress: GfsUploadProgress) => void,
onByteProgress?: (progress: GfsUploadProgress) => void,
+ onRetryableFailure?: (partNumber: number) => void,
onPartFailure?: (partNumber: number) => void,
fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY,
instabilityFailureThreshold = GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD
@@ -759,7 +775,10 @@ async function runParts(
parts.length,
signal,
onByteProgress,
- registerRetryableFailure,
+ () => {
+ registerRetryableFailure()
+ onRetryableFailure?.(part.partNumber)
+ },
registerPartSuccess
)
onProgress?.({
@@ -1267,6 +1286,17 @@ export class GfsUploadJob {
() => this.pauseRequested || this.canceled,
progress,
byteProgress,
+ partNumber => {
+ inFlightLoaded.delete(partNumber)
+ this.uploadedBytes = calculateUploadedBytes()
+ this.input.onProgress?.({
+ uploadedBytes: this.uploadedBytes,
+ totalBytes: this.input.file.size,
+ partNumber,
+ partCount: allParts.length,
+ })
+ this.emit()
+ },
partNumber => {
inFlightLoaded.delete(partNumber)
this.uploadedBytes = calculateUploadedBytes()
diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.ts
index fa12dbd6b..579816668 100644
--- a/desktop-app/src/__tests__/appService.gfs.test.ts
+++ b/desktop-app/src/__tests__/appService.gfs.test.ts
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { AppService, legacyEncodedFile, migrateDesktopGfsUploadState } from '../appService.js'
import { config, getActiveEnvKey } from '../config.js'
+import { DesktopUploadCapabilityError } from '../gfs/upload.js'
vi.mock('../chatStoreBinding.js', () => ({
bindChatStoreForUser: vi.fn(),
@@ -212,6 +213,10 @@ type UploadScopeTestService = {
scope: ReturnType<typeof scopedUploadRecord>['scope']
}
>
+ gfsClient: {
+ createResource: ReturnType<typeof vi.fn>
+ replaceFile: ReturnType<typeof vi.fn>
+ }
desktopGfsUploadStatePath: () => Promise<string>
startDesktopGfsUpload: ReturnType<typeof vi.fn>
authClient: {
@@ -225,12 +230,16 @@ type UploadScopeTestService = {
completePasswordLogin: (email: string, password: string) => ReturnType<AppService['googleLogin']>
googleLogin: AppService['googleLogin']
listGfsUploadSessions: AppService['listGfsUploadSessions']
+ getGfsUploadSnapshot: AppService['getGfsUploadSnapshot']
+ pauseGfsUpload: AppService['pauseGfsUpload']
+ cancelGfsUpload: AppService['cancelGfsUpload']
resumeGfsUpload: AppService['resumeGfsUpload']
logout: AppService['logout']
runScopedLegacyGfsUpload: <T>(
scope: ReturnType<typeof scopedUploadRecord>['scope'],
operation: (token: string, signal: AbortSignal) => Promise<T>
) => Promise<T>
+ startGfsFileUpload: AppService['startGfsFileUpload']
}
function authenticatedUploadService(statePath: string): UploadScopeTestService {
@@ -267,6 +276,116 @@ function authenticatedUploadService(statePath: string): UploadScopeTestService {
}
describe('AppService GFS upload security scope', () => {
+ it('falls back to the bounded legacy create path for a fresh start when v2 is disabled', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-legacy-start-fallback-'))
+ try {
+ const filePath = join(root, 'legacy.bin')
+ await writeFile(filePath, Buffer.from('legacy payload'))
+ const service = authenticatedUploadService(join(root, 'gfs-upload-sessions.json'))
+ service.startDesktopGfsUpload.mockRejectedValue(
+ new DesktopUploadCapabilityError('resumable uploads are disabled')
+ )
+ service.gfsClient = {
+ createResource: vi.fn().mockResolvedValue({
+ resourceId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ rid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ gfsUri: 'gfs://main/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ drive: 'main',
+ parentResourceId: 'parent-rid',
+ name: 'legacy.bin',
+ kind: 'file',
+ path: '/legacy.bin',
+ version: 1,
+ bytes: 14,
+ }),
+ replaceFile: vi.fn(),
+ }
+
+ const result = await service.startGfsFileUpload('parent-rid', 'legacy.bin', filePath, 'main')
+
+ expect(result).toMatchObject({
+ state: 'completed',
+ operation: 'create',
+ resultResourceId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ expectedBytes: 14,
+ })
+ expect(service.gfsClient.createResource).toHaveBeenCalledWith(
+ expect.objectContaining({
+ parentResourceId: 'parent-rid',
+ name: 'legacy.bin',
+ drive: 'main',
+ kind: 'file',
+ encodedData: Buffer.from('legacy payload').toString('base64'),
+ }),
+ 'token-a',
+ expect.any(AbortSignal)
+ )
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('keeps existing upload controls and read state available during a transient team hop', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-transient-controls-'))
+ try {
+ const statePath = join(root, 'gfs-upload-sessions.json')
+ const record = scopedUploadRecord('92929292-9292-4292-8292-929292929292')
+ await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))
+ const service = authenticatedUploadService(statePath) as UploadScopeTestService & {
+ gfsTransientTeamHopDepth: number
+ }
+ const job = {
+ snapshot: vi.fn(() => ({
+ state: 'uploading',
+ session: record.session,
+ uploadedBytes: 0,
+ totalBytes: record.fileSize,
+ })),
+ pause: vi.fn(async () => ({ ...record.session, state: 'paused' as const })),
+ cancel: vi.fn(async () => undefined),
+ }
+ service.gfsTransientTeamHopDepth = 1
+ service.gfsUploadJobs.set(record.uploadId, {
+ job,
+ promise: Promise.resolve(record.session),
+ scope: record.scope,
+ })
+
+ await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({
+ state: 'uploading',
+ })
+ await service.pauseGfsUpload(record.uploadId, 'main')
+ await service.cancelGfsUpload(record.uploadId, 'main')
+ expect(job.pause).toHaveBeenCalledTimes(1)
+ expect(job.cancel).toHaveBeenCalledTimes(1)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('maps a persisted active record to the renderer uploading state after restart', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-snapshot-state-'))
+ try {
+ const statePath = join(root, 'gfs-upload-sessions.json')
+ const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')
+ const service = authenticatedUploadService(statePath)
+ // The normal state loader converts orphaned active records to
+ // suspended_auth. This injects the already-migrated persisted branch so
+ // the IPC boundary itself is still proven not to leak `active`.
+ ;(service as any).readDesktopGfsUploadState = vi
+ .fn()
+ .mockResolvedValue({ version: 2, records: [record], quarantined: [] })
+
+ await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({
+ state: 'uploading',
+ uploadedBytes: 0,
+ totalBytes: 4,
+ })
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('quarantines legacy unscoped state instead of restoring it', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-state-migration-'))
try {
diff --git a/desktop-app/src/__tests__/appService.rpc.test.ts b/desktop-app/src/__tests__/appService.rpc.test.ts
index 26de93281..3242edb1f 100644
--- a/desktop-app/src/__tests__/appService.rpc.test.ts
+++ b/desktop-app/src/__tests__/appService.rpc.test.ts
@@ -250,6 +250,110 @@ describe('AppService.invokeHostMessage', () => {
expect(service.gfsTransientTeamHopDepth).toBe(0)
})
+ it('keeps existing GFS controls usable through the real team-hop lifecycle', async () => {
+ const service = new AppService() as any
+ const statePath = path.join(chatStoreBaseDir, 'gfs-upload-state.json')
+ const uploadId = '92929292-9292-4292-8292-929292929292'
+ const session = {
+ uploadId,
+ drive: 'main',
+ operation: 'create',
+ expectedBytes: 4,
+ partBytes: 4,
+ partCount: 1,
+ state: 'uploading',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ const scope = {
+ ownerId: 'user-1',
+ teamId: 'team-a',
+ environmentKey: 'test-env',
+ baseUrl: 'https://external.example',
+ drive: 'main',
+ authEpoch: 1,
+ }
+ await fs.writeFile(
+ statePath,
+ JSON.stringify({
+ version: 2,
+ records: [
+ {
+ version: 2,
+ uploadId,
+ filePath: '/tmp/payload.bin',
+ fileName: 'payload.bin',
+ fileSize: 4,
+ target: { operation: 'create', parentRid: 'parent' },
+ name: 'payload.bin',
+ session,
+ scope,
+ status: 'active',
+ updatedAt: new Date().toISOString(),
+ },
+ ],
+ quarantined: [],
+ })
+ )
+ service.desktopGfsUploadStatePath = vi.fn().mockResolvedValue(statePath)
+ service.sessionToken = 'team-a-token'
+ service.me = {
+ id: 'user-1',
+ email: '[email protected]',
+ name: 'Test User',
+ picture: null,
+ teamId: 'team-a',
+ teamName: 'Team A',
+ role: 'member',
+ }
+ service.gfsAuthEpoch = 1
+ service.gfsDispatchBlocked = false
+ service.gfsScopeIdentity = { ...scope, authEpoch: undefined }
+ const job = {
+ snapshot: vi.fn(() => ({ state: 'uploading', session, uploadedBytes: 0, totalBytes: 4 })),
+ pause: vi.fn(async () => ({ ...session, state: 'paused' })),
+ resume: vi.fn(async () => ({ ...session, state: 'uploading' })),
+ cancel: vi.fn(async () => undefined),
+ }
+ service.gfsUploadJobs = new Map([[uploadId, { job, promise: Promise.resolve(session), scope }]])
+ service.authClient = {
+ getMe: vi
+ .fn()
+ .mockResolvedValueOnce({ ...service.me, teamId: 'team-b', teamName: 'Team B' })
+ .mockResolvedValueOnce(service.me),
+ switchTeam: vi.fn(async (_token: string, teamId: string) => ({
+ token: `${teamId}-token`,
+ team: { id: teamId, name: teamId, role: 'member' },
+ })),
+ }
+ service.tokenStore = { setSessionToken: vi.fn() }
+
+ await service.runWithTeamContext('team-b', async () => {
+ await expect(service.getGfsUploadSnapshot(uploadId, 'main')).resolves.toMatchObject({
+ state: 'uploading',
+ })
+ await expect(service.listGfsUploadSessions('main')).resolves.toHaveLength(1)
+ await expect(service.pauseGfsUpload(uploadId, 'main')).resolves.toMatchObject({
+ state: 'paused',
+ })
+ await expect(service.resumeGfsUpload(uploadId, 'main')).resolves.toMatchObject({
+ state: 'uploading',
+ })
+ await expect(service.cancelGfsUpload(uploadId, 'main')).resolves.toBeUndefined()
+ })
+
+ expect(job.snapshot).toHaveBeenCalledTimes(1)
+ expect(job.pause).toHaveBeenCalledTimes(1)
+ expect(job.resume).toHaveBeenCalledTimes(1)
+ expect(job.cancel).toHaveBeenCalledTimes(1)
+ expect(service.me.teamId).toBe('team-a')
+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(1, 'team-a-token', 'team-b')
+ expect(service.authClient.switchTeam).toHaveBeenNthCalledWith(2, 'team-b-token', 'team-a')
+ })
+
it('fences the old GFS scope if a replacement team token cannot be refreshed', async () => {
const service = new AppService() as any
service.sessionToken = 'team-a-token'
diff --git a/desktop-app/src/appService.ts b/desktop-app/src/appService.ts
index 405e2fee6..79ce379b7 100644
--- a/desktop-app/src/appService.ts
+++ b/desktop-app/src/appService.ts
@@ -2136,14 +2136,33 @@ export class AppService {
drive?: string,
resumeUploadId?: string
): Promise<DesktopUploadSession> {
- return this.startDesktopGfsUpload({
- filePath,
- name,
- drive: normalizeDesktopUploadDrive(drive),
- operation: 'create',
- parentRid: parentResourceId,
- resumeUploadId,
- })
+ const canonicalDrive = normalizeDesktopUploadDrive(drive)
+ const scope = this.currentDesktopGfsUploadScope(canonicalDrive)
+ try {
+ return await this.startDesktopGfsUpload({
+ filePath,
+ name,
+ drive: canonicalDrive,
+ operation: 'create',
+ parentRid: parentResourceId,
+ resumeUploadId,
+ })
+ } catch (error) {
+ // A legacy fallback is valid only for a fresh upload. An explicit resume
+ // must remain a v2 operation so a missing capability cannot silently turn
+ // a persisted session into a second non-resumable resource.
+ if (!(error instanceof DesktopUploadCapabilityError) || resumeUploadId) throw error
+ const resource = await this.runScopedLegacyGfsUpload(scope, async (token, signal) => {
+ const encodedData = await legacyEncodedFile(filePath)
+ this.assertCurrentDesktopGfsUploadScope(scope)
+ return this.gfsClient.createResource(
+ { parentResourceId, drive: canonicalDrive, name, kind: 'file', encodedData },
+ token,
+ signal
+ )
+ })
+ return legacyReceipt(resource, 'create')
+ }
}
async startGfsFileReplace(
@@ -2153,20 +2172,39 @@ export class AppService {
ifMatch?: number,
resumeUploadId?: string
): Promise<DesktopUploadSession> {
- return this.startDesktopGfsUpload({
- filePath,
- name: resourceId,
- drive: normalizeDesktopUploadDrive(drive),
- operation: 'replace',
- resourceRid: resourceId,
- ifMatch,
- resumeUploadId,
- })
+ const canonicalDrive = normalizeDesktopUploadDrive(drive)
+ const scope = this.currentDesktopGfsUploadScope(canonicalDrive)
+ try {
+ return await this.startDesktopGfsUpload({
+ filePath,
+ name: resourceId,
+ drive: canonicalDrive,
+ operation: 'replace',
+ resourceRid: resourceId,
+ ifMatch,
+ resumeUploadId,
+ })
+ } catch (error) {
+ if (!(error instanceof DesktopUploadCapabilityError) || resumeUploadId) throw error
+ const resource = await this.runScopedLegacyGfsUpload(scope, async (token, signal) => {
+ const encodedData = await legacyEncodedFile(filePath)
+ this.assertCurrentDesktopGfsUploadScope(scope)
+ return this.gfsClient.replaceFile(
+ { resourceId, drive: canonicalDrive, ifMatch, encodedData },
+ token,
+ signal
+ )
+ })
+ return legacyReceipt(resource, 'replace')
+ }
}
async getGfsUploadSnapshot(uploadId: string, drive?: string) {
- const scope = this.currentDesktopGfsUploadScope(drive)
const entry = this.gfsUploadJobs.get(uploadId)
+ // Snapshot is read-only and an existing job already owns its captured
+ // token/scope. It must remain observable during a finite internal team hop;
+ // only a brand-new user-facing dispatch is fenced by that hop.
+ const scope = this.currentDesktopGfsUploadScope(drive, { allowTransientTeamHop: true })
if (entry && sameDesktopGfsUploadScope(entry.scope, scope)) return entry.job.snapshot()
const state = await this.readDesktopGfsUploadState()
const record = state.records.find(
@@ -2176,7 +2214,11 @@ export class AppService {
)
if (!record) return null
return {
- state: record.status,
+ // `active` is the persistence marker for a process-owned upload. The
+ // renderer contract describes the same resumable state as `uploading`;
+ // never leak the storage-only marker across the IPC boundary after a
+ // restart.
+ state: record.status === 'active' ? 'uploading' : record.status,
session: record.session,
uploadedBytes: record.session.committedBytes,
totalBytes: record.fileSize,
@@ -2184,8 +2226,10 @@ export class AppService {
}
async pauseGfsUpload(uploadId: string, drive?: string): Promise<DesktopUploadSession> {
- const scope = this.currentDesktopGfsUploadScope(drive)
const entry = this.gfsUploadJobs.get(uploadId)
+ const scope = this.currentDesktopGfsUploadScope(drive, {
+ allowTransientTeamHop: Boolean(entry),
+ })
if (!entry || !sameDesktopGfsUploadScope(entry.scope, scope))
throw new Error('GFS upload is not active in the current security scope')
const session = await entry.job.pause()
@@ -2201,8 +2245,10 @@ export class AppService {
}
async resumeGfsUpload(uploadId: string, drive?: string): Promise<DesktopUploadSession> {
- const scope = this.currentDesktopGfsUploadScope(drive)
const entry = this.gfsUploadJobs.get(uploadId)
+ const scope = this.currentDesktopGfsUploadScope(drive, {
+ allowTransientTeamHop: Boolean(entry),
+ })
if (entry) {
if (!sameDesktopGfsUploadScope(entry.scope, scope))
throw new Error('GFS upload is not active in the current security scope')
@@ -2231,8 +2277,10 @@ export class AppService {
}
async cancelGfsUpload(uploadId: string, drive?: string): Promise<void> {
- const scope = this.currentDesktopGfsUploadScope(drive)
const entry = this.gfsUploadJobs.get(uploadId)
+ const scope = this.currentDesktopGfsUploadScope(drive, {
+ allowTransientTeamHop: Boolean(entry),
+ })
if (entry) {
if (!sameDesktopGfsUploadScope(entry.scope, scope))
throw new Error('GFS upload is not active in the current security scope')
@@ -2261,7 +2309,9 @@ export class AppService {
}
async listGfsUploadSessions(drive?: string): Promise<DesktopGfsUploadSummary[]> {
- const scope = this.currentDesktopGfsUploadScope(drive)
+ // Listing persisted state is read-only and must remain available while a
+ // finite internal team hop temporarily borrows another team's token.
+ const scope = this.currentDesktopGfsUploadScope(drive, { allowTransientTeamHop: true })
const state = await this.readDesktopGfsUploadState()
return state.records
.filter(record => sameDesktopGfsUploadScope(record.scope, scope, { includeAuthEpoch: false }))
diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.ts
index 341ba7d5c..7ddaee2f3 100644
--- a/desktop-app/src/gfs/upload.test.ts
+++ b/desktop-app/src/gfs/upload.test.ts
@@ -239,6 +239,141 @@ describe('desktop GFS indexed uploader', () => {
}
})
+ it('uses the extended bounded part budget while a writer is restarting', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-writer-restart-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ await writeFile(filePath, Buffer.from([23]))
+ const statuses = [502, 502, 502, 502, 502, 204]
+ let partRequests = 0
+ const session = {
+ uploadId: '14141414-1414-4141-8141-141414141414',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: 1,
+ partBytes: 1,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'HEAD') return {} as T
+ if (method === 'GET' && requestPath(url).endsWith('/status'))
+ return { ok: true, data: { session, parts: [] } } as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'POST' && requestPath(url).endsWith('/complete'))
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ partRequests += 1
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* drain */
+ }
+ return { status: statuses.shift() ?? 204, text: '' }
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-writer-restart',
+ transport,
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+ expect(partRequests).toBe(6)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('removes a failed part attempt from aggregate progress before retrying it', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-progress-retry-'))
+ try {
+ const filePath = join(root, 'payload.bin')
+ await writeFile(filePath, Buffer.from([31, 32, 33, 34]))
+ const session = {
+ uploadId: '16161616-1616-4161-8161-161616161616',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: 4,
+ partBytes: 4,
+ partCount: 1,
+ state: 'initiated',
+ contiguousBytes: 0,
+ committedBytes: 0,
+ committedPartCount: 0,
+ activePartCount: 0,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+ let partRequests = 0
+ const progress: number[] = []
+ const transport = {
+ async requestJson<T>(method: 'GET' | 'POST' | 'HEAD', url: string): Promise<T> {
+ if (method === 'GET' && requestPath(url).endsWith('/capabilities'))
+ return enabledCapabilities() as T
+ if (method === 'POST' && requestPath(url).endsWith('/uploads'))
+ return { ok: true, data: session } as T
+ if (method === 'GET' && requestPath(url).endsWith('/status'))
+ return { ok: true, data: { session, parts: [] } } as T
+ if (method === 'POST' && requestPath(url).endsWith('/complete'))
+ return { ok: true, data: { ...session, state: 'completed' } } as T
+ if (method === 'HEAD') return {} as T
+ throw new Error(`unexpected ${method} ${url}`)
+ },
+ async requestPart(
+ _url: string,
+ _token: string,
+ _headers: Record<string, string>,
+ body: NodeJS.ReadableStream
+ ) {
+ for await (const _chunk of body as AsyncIterable<Buffer>) {
+ /* drain */
+ }
+ partRequests += 1
+ return { status: partRequests === 1 ? 502 : 204, text: '' }
+ },
+ }
+
+ await expect(
+ new DesktopGfsUploadJob({
+ baseUrl: 'https://api.example',
+ token: 'token',
+ filePath,
+ name: 'payload.bin',
+ drive: 'main',
+ operation: 'create',
+ parentRid: 'parent-progress-retry',
+ transport,
+ onProgress: (uploadedBytes: number) => progress.push(uploadedBytes),
+ }).start()
+ ).resolves.toMatchObject({ state: 'completed' })
+
+ expect(partRequests).toBe(2)
+ expect(progress.some((value, index) => index > 0 && value < progress[index - 1]!)).toBe(true)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('resumes from sparse status and does not replay a committed part', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-resume-'))
try {
diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.ts
index 1a579e165..5e510b8ca 100644
--- a/desktop-app/src/gfs/upload.ts
+++ b/desktop-app/src/gfs/upload.ts
@@ -14,6 +14,12 @@ const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
+/**
+ * A writer restart can keep the service endpoint unavailable for longer than
+ * the ordinary application-error retry budget. Keep this separate from the
+ * lifecycle budget so 502/503/504 recovery is longer but still bounded.
+ */
+export const GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS = 6
export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
@@ -263,6 +269,16 @@ function retryableError(error: unknown): boolean {
return status === undefined || retryable(status)
}
+function partRetryAttempts(error: unknown): number {
+ const status =
+ error && typeof error === 'object' && typeof (error as { status?: unknown }).status === 'number'
+ ? Number((error as { status: number }).status)
+ : undefined
+ return status === undefined || status === 502 || status === 503 || status === 504
+ ? GFS_UPLOAD_SERVICE_RETRY_MAX_ATTEMPTS
+ : GFS_UPLOAD_RETRY_MAX_ATTEMPTS
+}
+
/** Parse an RFC 7231 Retry-After value and cap it before it reaches a timer. */
export function parseRetryAfter(
value: string | null | undefined,
@@ -685,6 +701,7 @@ async function putPart(
onPartSuccess?: () => void
): Promise<void> {
let attempt = 0
+ let maxAttempts = GFS_UPLOAD_RETRY_MAX_ATTEMPTS
for (;;) {
assertAuthEpoch(input)
await assertSourceIdentity(input)
@@ -732,6 +749,7 @@ async function putPart(
} catch (error) {
if (signal?.aborted) throw signal.reason ?? error
if (!retryableError(error)) throw error
+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(error))
onRetryableFailure?.()
const reconciled = await reconcileAmbiguousPart(
input,
@@ -746,7 +764,7 @@ async function putPart(
onPartSuccess?.()
return
}
- if (attempt >= 2) throw error
+ if (attempt + 1 >= maxAttempts) throw error
await sleep(
retryAfterOf(error) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
signal
@@ -761,6 +779,7 @@ async function putPart(
if (!retryable(response.status)) {
throw uploadPartError(response.status, response.text)
}
+ maxAttempts = Math.max(maxAttempts, partRetryAttempts(response))
onRetryableFailure?.()
if (ambiguousPartResponse(response.status)) {
const reconciled = await reconcileAmbiguousPart(
@@ -777,7 +796,7 @@ async function putPart(
return
}
}
- if (attempt >= 2) throw uploadPartError(response.status, response.text)
+ if (attempt + 1 >= maxAttempts) throw uploadPartError(response.status, response.text)
await sleep(
retryAfterOf(response) ?? Math.min(1_000, GFS_UPLOAD_RETRY_BASE_DELAY_MS * 2 ** attempt),
signal
@@ -882,7 +901,11 @@ async function runParts(
)
input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)
},
- registerRetryableFailure,
+ () => {
+ registerRetryableFailure()
+ inFlightLoaded.delete(part.partNumber)
+ input.onProgress?.(calculateUploadedBytes(), session.expectedBytes)
+ },
registerPartSuccess
)
completedParts.add(part.partNumber)
diff --git a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
index 0e82e5f1f..09970a082 100644
--- a/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
+++ b/desktop-app/test/e2e-playwright/gfs-upload-v2.test.ts
@@ -14,8 +14,16 @@ import {
import {
GFS_UPLOAD_V2_BOUNDARIES,
createDiskUploadFixture,
+ createOversizedDiskUploadFixture,
removeDiskUploadFixture,
} from '../../../tests/e2e/gfsUploadV2Fixtures'
+import {
+ countGfsCreateSessions,
+ readGfsUploadV2Enabled,
+ restartGfsWriter,
+ revokeGfsUserWriteGrant,
+ setGfsUploadV2Enabled,
+} from '../../../tests/e2e/gfsUploadV2Runtime'
import { test } from './fixtures'
import { openResourcesNavItem } from './navigationHelpers'
@@ -413,3 +421,193 @@ test.describe('GFS Upload v2 — packaged Desktop project', () => {
})
}
})
+
+test.describe('GFS Upload v2 — approved negative Desktop journeys', () => {
+ test.skip(
+ process.env.GFS_UPLOAD_V2_NEGATIVE_E2E !== '1',
+ 'Set GFS_UPLOAD_V2_NEGATIVE_E2E=1 only on an owned non-production dev host with profile-scoped random URLs.'
+ )
+ test.setTimeout(45 * 60_000)
+
+ test('rejects a 209715201-byte file through the visible Desktop action', async ({ appPage }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-oversize-desktop')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ await seedGfsGrant({
+ resourceId: fixture.resourceId,
+ subjectType: 'user',
+ subjectId: getE2EUserId(),
+ permissions: ['read', 'write', 'delete'],
+ inherit: true,
+ grantedBy: 'e2e:gfs-upload-v2-negative',
+ })
+ const source = await createOversizedDiskUploadFixture('.parquet', fixtureName)
+ try {
+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)
+ await selectFileThroughVisibleAction(
+ appPage,
+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),
+ source.filePath
+ )
+ await expect(
+ appPage
+ .getByRole('alert')
+ .filter({ hasText: 'GFS uploads are limited to 200 MB per file.' })
+ ).toBeVisible({ timeout: 15_000 })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 15_000 }
+ )
+ .toBeNull()
+ await expect(browser.getByRole('button', { name: source.fileName, exact: true })).toHaveCount(
+ 0
+ )
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
+
+ test('falls back to the legacy path when v2 is disabled', async ({ appPage }) => {
+ const previous = await readGfsUploadV2Enabled()
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-legacy-desktop')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ await seedGfsGrant({
+ resourceId: fixture.resourceId,
+ subjectType: 'user',
+ subjectId: getE2EUserId(),
+ permissions: ['read', 'write', 'delete'],
+ inherit: true,
+ grantedBy: 'e2e:gfs-upload-v2-negative',
+ })
+ const source = await createDiskUploadFixture(2 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ await setGfsUploadV2Enabled(false)
+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)
+ await selectFileThroughVisibleAction(
+ appPage,
+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),
+ source.filePath
+ )
+ await expect(appPage.getByText(`Uploaded ${source.fileName}`, { exact: true })).toBeVisible({
+ timeout: 120_000,
+ })
+ expect(countGfsCreateSessions(fixture.resourceId, source.fileName, getE2EUserId())).toBe(0)
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })
+ await expect(
+ browser.getByRole('button', { name: source.fileName, exact: true })
+ ).toBeVisible()
+ } finally {
+ try {
+ await setGfsUploadV2Enabled(previous)
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ }
+ })
+
+ test('recovers a visible upload after the writer deployment restarts', async ({ appPage }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-restart-desktop')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ await seedGfsGrant({
+ resourceId: fixture.resourceId,
+ subjectType: 'user',
+ subjectId: getE2EUserId(),
+ permissions: ['read', 'write', 'delete'],
+ inherit: true,
+ grantedBy: 'e2e:gfs-upload-v2-negative',
+ })
+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)
+ await selectFileThroughVisibleAction(
+ appPage,
+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),
+ source.filePath
+ )
+ const progress = browser.getByRole('progressbar', {
+ name: `GFS upload progress for ${source.fileName}`,
+ })
+ await expect(progress).toBeVisible({ timeout: 60_000 })
+ await restartGfsWriter()
+ await expect(appPage.getByText(`Uploaded ${source.fileName}`, { exact: true })).toBeVisible({
+ timeout: 1_200_000,
+ })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toMatchObject({ kind: 'file', bytes: source.byteLength, deleted: false })
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
+
+ test('fails visibly when the user grant is revoked during an upload', async ({ appPage }) => {
+ const fixtureName = uniqueGfsFixtureName('e2e-gfs-v2-negative-revoke-desktop')
+ const fixture = seedGfsDirectoryFixture(fixtureName)
+ const userId = getE2EUserId()
+ seedGfsGrant({
+ resourceId: fixture.resourceId,
+ subjectType: 'user',
+ subjectId: userId,
+ permissions: ['read', 'write', 'delete'],
+ inherit: true,
+ grantedBy: 'e2e:gfs-upload-v2-negative',
+ })
+ const source = await createDiskUploadFixture(64 * 1024 * 1024, '.bin', fixtureName)
+ try {
+ const { browser, manageDialog } = await openFolder(appPage, fixture.name)
+ await selectFileThroughVisibleAction(
+ appPage,
+ manageDialog.getByRole('button').filter({ hasText: /^Upload file$/ }),
+ source.filePath
+ )
+ const progress = browser.getByRole('progressbar', {
+ name: `GFS upload progress for ${source.fileName}`,
+ })
+ await expect(progress).toBeVisible({ timeout: 60_000 })
+ await revokeGfsUserWriteGrant(fixture.resourceId, userId)
+ await expect(appPage.getByText('Upload needs attention', { exact: true })).toBeVisible({
+ timeout: 120_000,
+ })
+ await expect
+ .poll(
+ () =>
+ getGfsChildResourceSummary({
+ parentResourceId: fixture.resourceId,
+ name: source.fileName,
+ }),
+ { timeout: 60_000, intervals: [250, 1_000] }
+ )
+ .toBeNull()
+ } finally {
+ await removeDiskUploadFixture(source)
+ cleanupGfsFixture(fixtureName)
+ assertGfsFixtureCleaned(fixtureName)
+ }
+ })
+})
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
index 227cae0c8..20b486a26 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
@@ -128,6 +128,27 @@ function Probe() {
)
}
+function UploadProbe() {
+ const ctrl = useGfsBrowserController()
+ return (
+ <>
+ <div data-testid="upload-state">{ctrl.uploadSnapshot?.state ?? 'none'}</div>
+ <button
+ type="button"
+ onClick={() =>
+ void ctrl.startFileUpload({
+ parentResourceId: 'parent-rid',
+ name: 'legacy.bin',
+ filePath: '/tmp/legacy.bin',
+ })
+ }
+ >
+ start upload
+ </button>
+ </>
+ )
+}
+
function Harness({ children }: { children: ReactNode }) {
const [me, setMe] = useState<AuthContextValue['me']>(userA)
const [client] = useState(
@@ -191,6 +212,51 @@ describe('useGfsBrowserController', () => {
await waitFor(() => expect(screen.getByTestId('current').textContent).toBe('none'))
})
+ it('finishes a completed legacy receipt without polling it as a v2 session', async () => {
+ const startFileUpload = vi.fn(async () => ({
+ uploadId: 'legacy-resource-id',
+ drive: 'main',
+ operation: 'create' as const,
+ expectedBytes: 12,
+ partBytes: 12,
+ partCount: 1,
+ state: 'completed',
+ contiguousBytes: 12,
+ committedBytes: 12,
+ committedPartCount: 1,
+ activePartCount: 0,
+ expiresAt: new Date().toISOString(),
+ resultResourceId: 'legacy-resource-id',
+ resultVersion: 1,
+ }))
+ const getUploadSnapshot = vi.fn()
+ Object.defineProperty(window, 'clerum', {
+ configurable: true,
+ value: {
+ gfs: {
+ listAccessible: vi.fn(async () => ({ items: [], nextCursor: null })),
+ startFileUpload,
+ getUploadSnapshot,
+ },
+ },
+ })
+
+ render(<UploadProbe />, { wrapper: Harness })
+ await act(async () => {
+ screen.getByRole('button', { name: 'start upload' }).click()
+ })
+
+ await waitFor(() => expect(screen.getByTestId('upload-state').textContent).toBe('completed'))
+ expect(startFileUpload).toHaveBeenCalledWith(
+ 'parent-rid',
+ 'legacy.bin',
+ '/tmp/legacy.bin',
+ 'main',
+ undefined
+ )
+ expect(getUploadSnapshot).not.toHaveBeenCalled()
+ })
+
it('refreshes cached affordances after permissions change outside Desktop', async () => {
let held = ['read']
const affordances = vi.fn(async () => ({
diff --git a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
index 4e82d3602..3fb242271 100644
--- a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
+++ b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
@@ -300,12 +300,20 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {
input.resumeUploadId
)
uploadIdRef.current = session.uploadId
- setUploadSnapshot({
- state: session.state === 'paused' ? 'paused' : 'uploading',
+ const initialSnapshot: GfsUploadSnapshot = {
+ state: session.state as GfsUploadSnapshot['state'],
session,
uploadedBytes: session.committedBytes,
totalBytes: session.expectedBytes,
- })
+ }
+ setUploadSnapshot(initialSnapshot)
+ // The legacy compatibility path returns a completed resource receipt,
+ // not a resumable upload session. There is no v2 snapshot to poll under
+ // that resource id; refresh the folder and finish immediately.
+ if (session.state === 'completed') {
+ await refreshGfs()
+ return initialSnapshot
+ }
const snapshot = await waitForUpload(session.uploadId)
if (snapshot.state === 'completed') await refreshGfs()
return snapshot
@@ -328,12 +336,17 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {
input.resumeUploadId
)
uploadIdRef.current = session.uploadId
- setUploadSnapshot({
- state: session.state === 'paused' ? 'paused' : 'uploading',
+ const initialSnapshot: GfsUploadSnapshot = {
+ state: session.state as GfsUploadSnapshot['state'],
session,
uploadedBytes: session.committedBytes,
totalBytes: session.expectedBytes,
- })
+ }
+ setUploadSnapshot(initialSnapshot)
+ if (session.state === 'completed') {
+ await refreshGfs()
+ return initialSnapshot
+ }
const snapshot = await waitForUpload(session.uploadId)
if (snapshot.state === 'completed') await refreshGfs()
return snapshot
diff --git a/docs/architecture/gfs-upload-v2.md b/docs/architecture/gfs-upload-v2.md
index 7448e4f1f..f519c4429 100644
--- a/docs/architecture/gfs-upload-v2.md
+++ b/docs/architecture/gfs-upload-v2.md
@@ -39,6 +39,15 @@ finite and identical in Desktop and Control UI: `408`, `425`, `429`, `500`,
`502`, `503`, and `504`. `507 Insufficient Storage` and every other `5xx` are
terminal until an explicit server-side recovery/retry is requested. A server
may return `Retry-After`; clients honor it within their bounded retry budget.
+For binary part writes only, a transport failure or `502/503/504` receives a
+separate six-attempt service-recovery budget; ordinary lifecycle requests and
+other retryable statuses retain the three-attempt budget. This covers a bounded
+writer deployment restart without replaying a part blindly: each ambiguous
+attempt still reconciles the durable status and checksum first.
+The branch-owned Playwright gate can enable the negative journeys by setting
+`GFS_UPLOAD_V2_NEGATIVE_E2E=1`; they remain opt-in because they mutate HCC,
+restart the writer, and revoke only the seeded grant. The gate still requires
+an explicit non-production context and profile-owned random URLs.
## Edge admission limits
@@ -69,6 +78,12 @@ is unavailable. A denied budget returns stable
`429 { error: "gfs_upload_rate_limited", limit, retryAfterSeconds }` plus
`Retry-After`; both relays preserve that contract.
+The source-IP bucket is trusted only across the authenticated
+`external-rest-api` service boundary. Control API ignores a client-supplied
+`x-gfs-upload-source-ip` from any other caller and falls back to the socket or
+trusted proxy address, so a valid internal token cannot rotate the IP key to
+evade the per-IP budget.
+
## End-to-end shape
```text
@@ -122,6 +137,13 @@ false cancellation. The publication transaction re-checks state and epoch
before writing the resource, so a canceled response can never accompany a
published resource.
+Client progress is monotonic for stale `uploading` snapshots that arrive out
+of order. A failed part retry is different: its in-flight contribution is
+removed so the next aggregate may move downward to the truthful committed
+plus in-flight value. Completion is the only state that may expose the exact
+file size; the browser and Desktop tests cover both the stale-snapshot guard
+and this failure correction.
+
## Persistence and safety
Desktop upload persistence is a version-2 envelope. Every resumable record is
@@ -133,11 +155,15 @@ deliberate team switch, or runtime/API-base switch blocks new dispatch, advances
the epoch, aborts and awaits active v2 and legacy fallback work, clears
credentials, and persists the old records as local `suspended_auth`. A finite
internal team-context hop temporarily blocks only new GFS dispatches while it
-borrows the other team's token; existing uploads keep their captured token and
-scope, and the original team scope is restored before the gate opens. Resume is
-user-explicit and succeeds only when owner, team, environment, base URL, and
-drive match exactly; the new auth epoch is bound only after the server session
-is revalidated.
+borrows the other team's token; read-only listing/snapshot and controls for an
+existing job remain available, existing uploads keep their captured token and
+scope, and the original team scope is restored before the gate opens. A
+deliberate switch fences the old uploads after the replacement session
+succeeds; if that local fence cannot be persisted, the app fails closed by
+clearing the replacement session and credentials rather than leaving an
+ambiguous team/token pairing. Resume is user-explicit and succeeds only when
+owner, team, environment, base URL, and drive match exactly; the new auth epoch
+is bound only after the server session is revalidated.
The legacy compatibility fallback still accepts at most 16 MiB. It opens one
descriptor with no-follow semantics where available, validates the descriptor
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index c589ae108..e7eecf31b 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.69",
+ "version": "0.1.70",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.69",
+ "version": "0.1.70",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index b98e1c008..fe0cdaf72 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.69",
+ "version": "0.1.70",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.ts
index edf9c38a6..903f66fa4 100644
--- a/external-rest-api/src/app.ts
+++ b/external-rest-api/src/app.ts
@@ -21,7 +21,8 @@ import { createExternalWorkflowsRouter } from './routes/workflows.js'
// Upload v2 part bodies are streamed octets and must not pass through the
// global JSON parser (which would buffer/reject the binary payload).
-const GFS_UPLOAD_PART_PATH = /^\/api\/v1\/me\/gfs\/uploads\/[0-9a-f-]{36}\/parts\/[0-9]+$/i
+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/parts/[0-9]+$`, 'i')
export function createApp() {
const app = express()
diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.ts
index 8884ca6c1..40f76d293 100644
--- a/gfs-controller/src/api/serve.ts
+++ b/gfs-controller/src/api/serve.ts
@@ -178,7 +178,7 @@ const ACCESSIBLE_RE = /^\/v1\/accessible$/;
const COPY_RE = /^\/v1\/copy$/;
const CAPABILITIES_RE = /^\/v1\/capabilities$/;
const UPLOAD_COLLECTION_RE = /^\/v1\/uploads$/;
-const UPLOAD_ITEM_RE = /^\/v1\/uploads\/([0-9a-f-]{36})(?:\/(status|parts\/([0-9]+)|pause|resume|complete))?$/i;
+const UPLOAD_ITEM_RE = /^\/v1\/uploads\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\/(status|parts\/([0-9]+)|pause|resume|complete))?$/i;
const GFS_URI_RE = /^gfs:\/\/([^/]+)\/([^/?#]+)$/;
function bearerToken(req: IncomingMessage): string {
diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.ts
index 72a3c2b9a..af5ae9a0e 100644
--- a/gfs-controller/src/index.ts
+++ b/gfs-controller/src/index.ts
@@ -1,8 +1,7 @@
-import { createHash, randomUUID } from "node:crypto";
+import { randomUUID } from "node:crypto";
import { constants } from "node:fs";
import { lstat, mkdir, open, rm, stat } from "node:fs/promises";
import { resolve } from "node:path";
-import { Readable } from "node:stream";
import { Client, Pool } from "pg";
import { verifyGfsToken } from "./auth/verify";
import { KeyConfigError, loadVerificationKey } from "./auth/keys";
@@ -20,46 +19,14 @@ import { PgBlobStagingStore, reconcileExpiredBlobs } from "./db/blobStaging";
import { GfsMetrics } from "./metrics";
import { GfsServer, ReadinessDeps } from "./server";
import { BlobStore } from "./storage/blobStore";
-import { GfsUploadSessionService, uploadCapabilities, type UploadPartRow, type UploadSessionRow } from "./upload/uploadSession";
+import { GfsUploadSessionService, uploadCapabilities } from "./upload/uploadSession";
+import { createGfsUploadFinalizer } from "./upload/uploadFinalizer";
import { GfsError } from "./api/errors";
function errMsg(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
-function orderedUploadStream(storageMountPath: string, parts: readonly UploadPartRow[], signal?: AbortSignal): Readable {
- return Readable.from((async function* () {
- for (const part of [...parts].sort((a, b) => a.partNumber - b.partNumber)) {
- signal?.throwIfAborted();
- const root = resolve(storageMountPath);
- const expectedPath = resolve(root, ".uploads", part.uploadId, "parts", `${part.partNumber}.part`);
- if (part.stagingPath !== expectedPath || !part.stagingPath.startsWith(`${root}/.uploads/${part.uploadId}/parts/`)) {
- throw new GfsError("path_invalid", "staged upload part path is invalid");
- }
- const handle = await open(part.stagingPath, constants.O_RDONLY | constants.O_NOFOLLOW);
- const buffer = Buffer.allocUnsafe(64 * 1024);
- const digest = createHash("sha256");
- let bytes = 0;
- try {
- for (;;) {
- signal?.throwIfAborted();
- const result = await handle.read(buffer, 0, buffer.length, null);
- if (result.bytesRead === 0) break;
- const chunk = Buffer.from(buffer.subarray(0, result.bytesRead));
- bytes += result.bytesRead;
- digest.update(chunk);
- yield chunk;
- }
- } finally {
- await handle.close().catch(() => undefined);
- }
- if (bytes !== part.lengthBytes || digest.digest("hex") !== part.sha256) {
- throw new GfsError("checksum_mismatch", `staged upload part ${part.partNumber} failed verification`);
- }
- }
- })());
-}
-
async function assertUploadV2Ready(
pool: Pool,
storageMountPath: string,
@@ -293,76 +260,13 @@ async function main(): Promise<void> {
const decision = await permissions.authorize(targetContext, targetRid, "write");
if (!decision.allowed) throw new GfsError("forbidden", `not authorized to ${operation} upload`);
},
- finalize: async (session: UploadSessionRow, parts: UploadPartRow[], signal?: AbortSignal, deadlineAtMs?: number) => {
- const capturedPermissionEpoch = permissions.permissionEpoch();
- const source = {
- stream: orderedUploadStream(config.storageMountPath, parts, signal),
- expectedBytes: session.expectedBytes,
- ...(session.wholeSha256 ? { expectedSha256: session.wholeSha256 } : {}),
- };
- const mutation = { subject: session.primarySubject, requestId: session.uploadId, audit };
- const onPublished = async (client: import("./db/writeStore").TxClient, published: import("./api/read").GfsResource, publicationSignal?: AbortSignal): Promise<void> => {
- publicationSignal?.throwIfAborted();
- if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {
- throw new GfsError("precondition_failed", "upload finalization timed out");
- }
- const currentPermissionEpoch = permissions.permissionEpoch();
- if (
- capturedPermissionEpoch.bypassed
- || currentPermissionEpoch.bypassed
- || currentPermissionEpoch.generation !== capturedPermissionEpoch.generation
- ) {
- throw new GfsError("precondition_failed", "authorization changed during upload finalization");
- }
- const targetRid = session.operation === "create" ? session.parentRid : session.resourceRid;
- if (!targetRid) throw new GfsError("path_invalid", "upload target is required");
- const targetContext = await resolveAuthzContext(pool, {
- sub: session.ownerSubject,
- drive: session.drive,
- });
- const decision = await permissions.authorize(targetContext, targetRid, "write");
- publicationSignal?.throwIfAborted();
- if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {
- throw new GfsError("precondition_failed", "upload finalization timed out");
- }
- if (!decision.allowed) throw new GfsError("forbidden", "upload authorization was revoked before publication");
- const completed = await client.query(
- `UPDATE gfs_upload_sessions
- SET state = 'completed', result_resource_id = $2, result_version = $3,
- result_sha256 = $4, completed_at = now(), updated_at = now()
- WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $5
- RETURNING upload_id`,
- [session.uploadId, published.resourceId, published.version, published.contentSha256, session.sessionEpoch]
- );
- publicationSignal?.throwIfAborted();
- if (completed.rows.length !== 1) throw new GfsError("upload_aborted", "upload session changed before publication");
- };
- const resource = session.operation === "create"
- ? await writeService.create({
- drive: session.drive,
- parentId: session.parentRid!,
- name: session.resourceName!,
- kind: "file",
- resourceId: session.uploadId,
- content: source,
- mutation,
- onPublished,
- signal,
- deadlineAtMs,
- })
- : await writeService.replace({
- drive: session.drive,
- resourceId: session.resourceRid!,
- ifMatch: session.ifMatch ?? undefined,
- content: source,
- mutation,
- onPublished,
- signal,
- deadlineAtMs,
- });
- if (!resource.contentSha256) throw new GfsError("internal", "published upload has no checksum");
- return { resourceId: resource.resourceId, version: resource.version, sha256: resource.contentSha256 };
- },
+ finalize: createGfsUploadFinalizer({
+ pool,
+ storageMountPath: config.storageMountPath,
+ permissions,
+ writeService,
+ audit,
+ }),
})),
} : {}),
});
diff --git a/gfs-controller/src/upload/protocol.test.ts b/gfs-controller/src/upload/protocol.test.ts
index 2a0b47352..f10911d68 100644
--- a/gfs-controller/src/upload/protocol.test.ts
+++ b/gfs-controller/src/upload/protocol.test.ts
@@ -1,13 +1,13 @@
import { describe, expect, it } from 'vitest'
import {
- disabledGfsUploadV2Capability,
GFS_UPLOAD_V2_DEFAULT_CONCURRENCY,
GFS_UPLOAD_V2_FALLBACK_CONCURRENCY,
+ GFS_UPLOAD_V2_INSTABILITY_FAILURE_THRESHOLD,
GFS_UPLOAD_V2_MAX_PART_BYTES,
+ GFS_UPLOAD_V2_PREFERRED_PART_BYTES,
GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,
GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES,
- GFS_UPLOAD_V2_PREFERRED_PART_BYTES,
- GFS_UPLOAD_V2_INSTABILITY_FAILURE_THRESHOLD,
+ disabledGfsUploadV2Capability,
isGfsUploadV2Route,
partCountFor,
partGeometry,
@@ -32,9 +32,16 @@ describe('GFS Upload v2 frozen contract', () => {
})
it('calculates exact 8 MiB geometry at the product boundary', () => {
- const geometry = { expectedBytes: GFS_UPLOAD_V2_PRODUCT_MAX_BYTES, partBytes: GFS_UPLOAD_V2_PREFERRED_PART_BYTES }
+ const geometry = {
+ expectedBytes: GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,
+ partBytes: GFS_UPLOAD_V2_PREFERRED_PART_BYTES,
+ }
expect(partCountFor(geometry)).toBe(25)
- expect(partGeometry(geometry, 0)).toEqual({ partNumber: 0, offsetBytes: 0, lengthBytes: 8_388_608 })
+ expect(partGeometry(geometry, 0)).toEqual({
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: 8_388_608,
+ })
expect(partGeometry(geometry, 24)).toEqual({
partNumber: 24,
offsetBytes: 201_326_592,
@@ -55,11 +62,40 @@ describe('GFS Upload v2 frozen contract', () => {
)
})
+ it.each([
+ [0, 1 * 1024 * 1024],
+ [1, 1 * 1024 * 1024],
+ [1_048_577, 1 * 1024 * 1024],
+ [16 * 1024 * 1024 + 1, 8 * 1024 * 1024],
+ [209_715_200, 16 * 1024 * 1024],
+ ])('keeps generated geometry contiguous for expected=%d part=%d', (expectedBytes, partBytes) => {
+ const geometry = { expectedBytes, partBytes }
+ const count = partCountFor(geometry)
+ const parts = Array.from({ length: count }, (_, partNumber) =>
+ partGeometry(geometry, partNumber)
+ )
+ expect(parts.reduce((sum, part) => sum + part.lengthBytes, 0)).toBe(expectedBytes)
+ for (let index = 0; index < parts.length; index += 1) {
+ const part = parts[index]!
+ expect(part.offsetBytes).toBe(
+ index === 0 ? 0 : parts[index - 1]!.offsetBytes + parts[index - 1]!.lengthBytes
+ )
+ expect(part.lengthBytes).toBeGreaterThan(0)
+ expect(part.lengthBytes).toBeLessThanOrEqual(partBytes)
+ validatePartGeometry(geometry, part)
+ }
+ })
+
it('recognizes only the exact v2 routes', () => {
expect(isGfsUploadV2Route('/v1/capabilities')).toBe(true)
expect(isGfsUploadV2Route('/v1/uploads')).toBe(true)
- expect(isGfsUploadV2Route('/v1/uploads/01234567-89ab-cdef-0123-456789abcdef/parts/0')).toBe(true)
+ expect(isGfsUploadV2Route('/v1/uploads/01234567-89ab-cdef-0123-456789abcdef/parts/0')).toBe(
+ true
+ )
expect(isGfsUploadV2Route('/v1/resources/0123456789abcdef0123456789abcdef')).toBe(false)
expect(isGfsUploadV2Route('/v1/uploads/not-an-id/parts/0')).toBe(false)
+ expect(isGfsUploadV2Route('/v1/uploads/------------------------------------/parts/0')).toBe(
+ false
+ )
})
})
diff --git a/gfs-controller/src/upload/protocol.ts b/gfs-controller/src/upload/protocol.ts
index 06cbc95d3..4be93c216 100644
--- a/gfs-controller/src/upload/protocol.ts
+++ b/gfs-controller/src/upload/protocol.ts
@@ -1,10 +1,12 @@
/**
* Shared, side-effect-free constants and geometry helpers for GFS Upload v2.
*
- * The writer and both clients import this module so the capability response,
- * session admission, part routes, and UI geometry share one arithmetic
- * definition. The route remains capability-gated: disabled deployments still
- * advertise the legacy surface and reject v2 mutations.
+ * The writer owns the authoritative arithmetic used by the capability
+ * response, session admission, and part routes. The separately packaged web
+ * and Desktop clients mirror the wire values at their boundaries rather than
+ * importing writer code; contract tests pin those copies to this definition.
+ * The route remains capability-gated: disabled deployments still advertise the
+ * legacy surface and reject v2 mutations.
*/
export const GFS_UPLOAD_V2_PRODUCT_MAX_BYTES = 209_715_200
@@ -23,6 +25,8 @@ export const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000
export const GFS_UPLOAD_V2_FINALIZE_TIMEOUT_MS = 600_000
export const GFS_UPLOAD_V2_STALE_PART_LEASE_MS = 600_000
+const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+
export type GfsUploadOperation = 'create' | 'replace'
export interface GfsUploadGeometry {
@@ -63,7 +67,11 @@ export function partCountFor({ expectedBytes, partBytes }: GfsUploadGeometry): n
}
export function validateGeometry({ expectedBytes, partBytes }: GfsUploadGeometry): void {
- if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0 || expectedBytes > GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES) {
+ if (
+ !Number.isSafeInteger(expectedBytes) ||
+ expectedBytes < 0 ||
+ expectedBytes > GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES
+ ) {
throw new GfsUploadGeometryError(
'invalid_expected_bytes',
`expectedBytes must be an integer from 0 through ${GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES}`
@@ -113,13 +121,22 @@ export function validatePartGeometry(
part: GfsUploadPartGeometry
): void {
if (!Number.isSafeInteger(part.partNumber) || part.partNumber < 0) {
- throw new GfsUploadGeometryError('invalid_part_number', 'partNumber must be a non-negative integer')
+ throw new GfsUploadGeometryError(
+ 'invalid_part_number',
+ 'partNumber must be a non-negative integer'
+ )
}
if (!Number.isSafeInteger(part.offsetBytes) || part.offsetBytes < 0) {
- throw new GfsUploadGeometryError('invalid_part_offset', 'offsetBytes must be a non-negative integer')
+ throw new GfsUploadGeometryError(
+ 'invalid_part_offset',
+ 'offsetBytes must be a non-negative integer'
+ )
}
if (!Number.isSafeInteger(part.lengthBytes) || part.lengthBytes <= 0) {
- throw new GfsUploadGeometryError('invalid_part_length', 'lengthBytes must be a positive integer')
+ throw new GfsUploadGeometryError(
+ 'invalid_part_length',
+ 'lengthBytes must be a positive integer'
+ )
}
const expected = partGeometry(geometry, part.partNumber)
if (part.offsetBytes !== expected.offsetBytes) {
@@ -141,7 +158,12 @@ export function isGfsUploadV2Route(pathname: string): boolean {
return (
pathname === '/v1/capabilities' ||
pathname === '/v1/uploads' ||
- /^\/v1\/uploads\/[0-9a-f-]{36}(?:\/status|\/parts\/[0-9]+|\/pause|\/resume|\/complete)?$/.test(pathname)
+ new RegExp(
+ `^/v1/uploads/${UUID_PATH}(?:/status|/parts/[0-9]+|/pause|/resume|/complete)?$`,
+ 'i'
+ ).test(
+ pathname
+ )
)
}
diff --git a/gfs-controller/src/upload/uploadFinalizer.test.ts b/gfs-controller/src/upload/uploadFinalizer.test.ts
new file mode 100644
index 000000000..91bef2e24
--- /dev/null
+++ b/gfs-controller/src/upload/uploadFinalizer.test.ts
@@ -0,0 +1,171 @@
+import { describe, expect, it, vi } from 'vitest'
+import { createHash } from 'node:crypto'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { DbAuditSink, PermissionClient } from '../authz/permissionClient'
+import { createGfsUploadFinalizer } from './uploadFinalizer'
+import type { UploadPartRow, UploadSessionRow } from './uploadSession'
+
+const UPLOAD_ID = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
+const PARENT_ID = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'
+const BODY = Buffer.from('published upload')
+const SHA256 = createHash('sha256').update(BODY).digest('hex')
+
+function session(): UploadSessionRow {
+ return {
+ uploadId: UPLOAD_ID,
+ idempotencyKey: 'finalizer-test',
+ drive: 'main',
+ ownerSubject: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
+ primarySubject: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
+ operation: 'create',
+ requestFingerprint: 'fingerprint',
+ parentRid: PARENT_ID,
+ resourceRid: null,
+ resourceName: 'payload.bin',
+ ifMatch: null,
+ expectedBytes: BODY.length,
+ partBytes: BODY.length,
+ partCount: 1,
+ wholeSha256: SHA256,
+ committedBytes: BODY.length,
+ contiguousBytes: BODY.length,
+ committedPartCount: 1,
+ activePartCount: 0,
+ sessionEpoch: 0,
+ state: 'finalizing',
+ resultResourceId: null,
+ resultVersion: null,
+ resultSha256: null,
+ failureCode: null,
+ expiresAt: new Date(Date.now() + 60_000).toISOString(),
+ }
+}
+
+function part(root: string): UploadPartRow {
+ return {
+ uploadId: UPLOAD_ID,
+ partNumber: 0,
+ offsetBytes: 0,
+ lengthBytes: BODY.length,
+ sha256: SHA256,
+ state: 'committed',
+ stagingPath: join(root, '.uploads', UPLOAD_ID, 'parts', '0.part'),
+ leaseEpoch: 0,
+ }
+}
+
+function finalizerHarness(
+ root: string,
+ permissionEpoch: () => { generation: number; bypassed: boolean }
+) {
+ const permissions = {
+ permissionEpoch,
+ authorize: vi.fn().mockResolvedValue({ allowed: true }),
+ } as unknown as PermissionClient
+ const client = {
+ query: vi.fn().mockResolvedValue({ rows: [{}] }),
+ }
+ const resource = {
+ resourceId: UPLOAD_ID,
+ rid: UPLOAD_ID.replaceAll('-', ''),
+ drive: 'main',
+ parentResourceId: PARENT_ID,
+ name: 'payload.bin',
+ kind: 'file',
+ bytes: BODY.length,
+ version: 0,
+ blobKey: 'generation/payload',
+ contentSha256: SHA256,
+ deletedAt: null,
+ path: '/payload.bin',
+ updatedAt: new Date().toISOString(),
+ }
+ const writeService = {
+ create: vi.fn(async (input: any) => {
+ const chunks: Buffer[] = []
+ for await (const chunk of input.content.stream) chunks.push(Buffer.from(chunk))
+ expect(Buffer.concat(chunks)).toEqual(BODY)
+ await input.onPublished(client, resource)
+ return resource
+ }),
+ replace: vi.fn(),
+ }
+ const pool = { query: vi.fn().mockResolvedValue({ rows: [] }) }
+ const finalizer = createGfsUploadFinalizer({
+ pool: pool as never,
+ storageMountPath: root,
+ permissions,
+ writeService: writeService as never,
+ audit: {} as DbAuditSink,
+ })
+ return { finalizer, permissions, client, writeService }
+}
+
+describe('production GFS upload finalizer seam', () => {
+ it('streams committed parts and marks the session completed in the publication transaction', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-'))
+ try {
+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')
+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })
+ await writeFile(partPath, BODY)
+ const harness = finalizerHarness(root, () => ({ generation: 1, bypassed: false }))
+
+ await expect(harness.finalizer(session(), [part(root)])).resolves.toEqual({
+ resourceId: UPLOAD_ID,
+ version: 0,
+ sha256: SHA256,
+ })
+ expect(harness.client.query).toHaveBeenCalledWith(
+ expect.stringContaining("SET state = 'completed'"),
+ expect.arrayContaining([UPLOAD_ID, UPLOAD_ID, 0, SHA256, 0])
+ )
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('rejects a permission-epoch change before publication and never marks completion', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-revoked-'))
+ try {
+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')
+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })
+ await writeFile(partPath, BODY)
+ let generation = 1
+ const harness = finalizerHarness(root, () => {
+ return { generation, bypassed: false }
+ })
+ harness.permissions.authorize = vi.fn().mockImplementation(async () => {
+ generation = 2
+ return { allowed: true }
+ }) as never
+
+ await expect(harness.finalizer(session(), [part(root)])).rejects.toMatchObject({
+ code: 'precondition_failed',
+ message: 'authorization changed during upload finalization',
+ })
+ expect(harness.client.query).not.toHaveBeenCalled()
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it('rejects an explicit authorization denial before publication', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'gfs-upload-finalizer-denied-'))
+ try {
+ const partPath = join(root, '.uploads', UPLOAD_ID, 'parts', '0.part')
+ await mkdir(join(root, '.uploads', UPLOAD_ID, 'parts'), { recursive: true })
+ await writeFile(partPath, BODY)
+ const harness = finalizerHarness(root, () => ({ generation: 1, bypassed: false }))
+ harness.permissions.authorize = vi.fn().mockResolvedValue({ allowed: false }) as never
+
+ await expect(harness.finalizer(session(), [part(root)])).rejects.toMatchObject({
+ code: 'forbidden',
+ })
+ expect(harness.client.query).not.toHaveBeenCalled()
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+})
diff --git a/gfs-controller/src/upload/uploadFinalizer.ts b/gfs-controller/src/upload/uploadFinalizer.ts
new file mode 100644
index 000000000..a25009c6f
--- /dev/null
+++ b/gfs-controller/src/upload/uploadFinalizer.ts
@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto'
+import { constants } from 'node:fs'
+import { open } from 'node:fs/promises'
+import { resolve } from 'node:path'
+import { Readable } from 'node:stream'
+import type { Pool } from 'pg'
+import { GfsError } from '../api/errors'
+import type { GfsResource } from '../api/read'
+import type { DbAuditSink, PermissionClient } from '../authz/permissionClient'
+import { resolveAuthzContext } from '../authz/subjectResolver'
+import { GfsWriteService, type TxClient } from '../db/writeStore'
+import type { UploadPartRow, UploadSessionRow } from './uploadSession'
+
+function orderedUploadStream(
+ storageMountPath: string,
+ parts: readonly UploadPartRow[],
+ signal?: AbortSignal
+): Readable {
+ return Readable.from(
+ (async function* () {
+ for (const part of [...parts].sort((a, b) => a.partNumber - b.partNumber)) {
+ signal?.throwIfAborted()
+ const root = resolve(storageMountPath)
+ const expectedPath = resolve(
+ root,
+ '.uploads',
+ part.uploadId,
+ 'parts',
+ `${part.partNumber}.part`
+ )
+ if (
+ part.stagingPath !== expectedPath ||
+ !part.stagingPath.startsWith(`${root}/.uploads/${part.uploadId}/parts/`)
+ ) {
+ throw new GfsError('path_invalid', 'staged upload part path is invalid')
+ }
+ const handle = await open(part.stagingPath, constants.O_RDONLY | constants.O_NOFOLLOW)
+ const buffer = Buffer.allocUnsafe(64 * 1024)
+ const digest = createHash('sha256')
+ let bytes = 0
+ try {
+ for (;;) {
+ signal?.throwIfAborted()
+ const result = await handle.read(buffer, 0, buffer.length, null)
+ if (result.bytesRead === 0) break
+ const chunk = Buffer.from(buffer.subarray(0, result.bytesRead))
+ bytes += result.bytesRead
+ digest.update(chunk)
+ yield chunk
+ }
+ } finally {
+ await handle.close().catch(() => undefined)
+ }
+ if (bytes !== part.lengthBytes || digest.digest('hex') !== part.sha256) {
+ throw new GfsError(
+ 'checksum_mismatch',
+ `staged upload part ${part.partNumber} failed verification`
+ )
+ }
+ }
+ })()
+ )
+}
+
+export interface GfsUploadFinalizerDeps {
+ pool: Pool
+ storageMountPath: string
+ permissions: PermissionClient
+ writeService: Pick<GfsWriteService, 'create' | 'replace'>
+ audit: DbAuditSink
+}
+
+/**
+ * Production upload finalizer seam. Keeping this closure in a small module
+ * makes the writer's stream, permission-epoch recheck, and same-transaction
+ * receipt update executable in focused tests instead of hiding them behind
+ * the process bootstrap.
+ */
+export function createGfsUploadFinalizer(
+ deps: GfsUploadFinalizerDeps
+): (
+ session: UploadSessionRow,
+ parts: UploadPartRow[],
+ signal?: AbortSignal,
+ deadlineAtMs?: number
+) => Promise<{ resourceId: string; version: number; sha256: string }> {
+ return async (session, parts, signal, deadlineAtMs) => {
+ const capturedPermissionEpoch = deps.permissions.permissionEpoch()
+ const source = {
+ stream: orderedUploadStream(deps.storageMountPath, parts, signal),
+ expectedBytes: session.expectedBytes,
+ ...(session.wholeSha256 ? { expectedSha256: session.wholeSha256 } : {}),
+ }
+ const mutation = {
+ subject: session.primarySubject,
+ requestId: session.uploadId,
+ audit: deps.audit,
+ }
+ const onPublished = async (
+ client: TxClient,
+ published: GfsResource,
+ publicationSignal?: AbortSignal
+ ): Promise<void> => {
+ publicationSignal?.throwIfAborted()
+ if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {
+ throw new GfsError('precondition_failed', 'upload finalization timed out')
+ }
+ const currentPermissionEpoch = deps.permissions.permissionEpoch()
+ if (
+ capturedPermissionEpoch.bypassed ||
+ currentPermissionEpoch.bypassed ||
+ currentPermissionEpoch.generation !== capturedPermissionEpoch.generation
+ ) {
+ throw new GfsError(
+ 'precondition_failed',
+ 'authorization changed during upload finalization'
+ )
+ }
+ const targetRid = session.operation === 'create' ? session.parentRid : session.resourceRid
+ if (!targetRid) throw new GfsError('path_invalid', 'upload target is required')
+ const targetContext = await resolveAuthzContext(deps.pool, {
+ sub: session.ownerSubject,
+ drive: session.drive,
+ })
+ const decision = await deps.permissions.authorize(targetContext, targetRid, 'write')
+ publicationSignal?.throwIfAborted()
+ if (deadlineAtMs !== undefined && Date.now() >= deadlineAtMs) {
+ throw new GfsError('precondition_failed', 'upload finalization timed out')
+ }
+ const postAuthorizationEpoch = deps.permissions.permissionEpoch()
+ if (
+ postAuthorizationEpoch.bypassed ||
+ postAuthorizationEpoch.generation !== capturedPermissionEpoch.generation
+ ) {
+ throw new GfsError(
+ 'precondition_failed',
+ 'authorization changed during upload finalization'
+ )
+ }
+ if (!decision.allowed) {
+ throw new GfsError('forbidden', 'upload authorization was revoked before publication')
+ }
+ const completed = await client.query(
+ `UPDATE gfs_upload_sessions
+ SET state = 'completed', result_resource_id = $2, result_version = $3,
+ result_sha256 = $4, completed_at = now(), updated_at = now()
+ WHERE upload_id = $1 AND state = 'finalizing' AND session_epoch = $5
+ RETURNING upload_id`,
+ [
+ session.uploadId,
+ published.resourceId,
+ published.version,
+ published.contentSha256,
+ session.sessionEpoch,
+ ]
+ )
+ publicationSignal?.throwIfAborted()
+ if (completed.rows.length !== 1) {
+ throw new GfsError('upload_aborted', 'upload session changed before publication')
+ }
+ }
+ const resource =
+ session.operation === 'create'
+ ? await deps.writeService.create({
+ drive: session.drive,
+ parentId: session.parentRid!,
+ name: session.resourceName!,
+ kind: 'file',
+ resourceId: session.uploadId,
+ content: source,
+ mutation,
+ onPublished,
+ signal,
+ deadlineAtMs,
+ })
+ : await deps.writeService.replace({
+ drive: session.drive,
+ resourceId: session.resourceRid!,
+ ifMatch: session.ifMatch ?? undefined,
+ content: source,
+ mutation,
+ onPublished,
+ signal,
+ deadlineAtMs,
+ })
+ if (!resource.contentSha256) throw new GfsError('internal', 'published upload has no checksum')
+ return {
+ resourceId: resource.resourceId,
+ version: resource.version,
+ sha256: resource.contentSha256,
+ }
+ }
+}
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index bb561f9ee..6b0476048 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -697,6 +697,18 @@ describe('GfsUploadSessionService', () => {
(uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
).toBe(0)
+ // The first reconciliation attempt must exercise the terminal-session
+ // cancellation fence, then release it even though filesystem cleanup is
+ // still blocked by the non-directory path. This catches a vacuous
+ // assertion before reconcile() and proves the fence is bounded across
+ // repeated cleanup cycles.
+ await expect(uploads.reconcile()).resolves.toEqual({ staleParts: 0, expiredSessions: 0 })
+ expect(
+ (uploads as unknown as { canceledUploads: Set<string> }).canceledUploads.has(
+ created.session.uploadId
+ )
+ ).toBe(false)
+
await rm(uploadDirectory, { force: true })
await mkdir(uploadDirectory, { recursive: true })
await writeFile(join(uploadDirectory, 'finalizer-marker'), 'pending')
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index e094cfb0a..c0d8e73f8 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -1354,6 +1354,12 @@ export class GfsUploadSessionService {
)
})
.catch(() => undefined)
+ // A terminal session with no registered flight no longer needs the
+ // in-memory cancellation fence. Keep it only while an operation is
+ // still alive so repeated filesystem failures cannot leak one set
+ // entry per terminal upload; the durable terminal row remains the
+ // authority for the next reconciliation cycle.
+ if (!this.hasInFlight(uploadId)) this.canceledUploads.delete(uploadId)
}
}
return { staleParts, expiredSessions }
diff --git a/scripts/e2e/gfs-upload-v2-gate.sh b/scripts/e2e/gfs-upload-v2-gate.sh
index 7665f9ac4..769bd38b6 100755
--- a/scripts/e2e/gfs-upload-v2-gate.sh
+++ b/scripts/e2e/gfs-upload-v2-gate.sh
@@ -26,22 +26,37 @@ python3 tools/e2e_static_audit.py \
run_ui="${GFS_UPLOAD_V2_RUN_UI:-0}"
run_desktop="${GFS_UPLOAD_V2_RUN_DESKTOP:-0}"
run_integration="${GFS_UPLOAD_V2_RUN_INTEGRATION:-0}"
+# Negative journeys mutate HCC, restart the writer, and revoke only the seeded
+# grant. Keep them explicit so a caller cannot turn an arbitrary context into
+# a destructive default by selecting a UI lane alone.
+run_negative="${GFS_UPLOAD_V2_NEGATIVE_E2E:-0}"
if [[ "$run_ui" == "1" ]]; then
- GFS_UPLOAD_V2_E2E=1 npm --prefix control-ui exec playwright test \
+ GFS_UPLOAD_V2_E2E=1 GFS_UPLOAD_V2_NEGATIVE_E2E="$run_negative" npm --prefix control-ui exec playwright test \
--config=playwright.config.ts --project=large-upload e2e/gfs-upload-v2.spec.ts
fi
if [[ "$run_desktop" == "1" ]]; then
- GFS_UPLOAD_V2_E2E=1 npm --prefix desktop-app exec playwright test \
+ GFS_UPLOAD_V2_E2E=1 GFS_UPLOAD_V2_NEGATIVE_E2E="$run_negative" npm --prefix desktop-app exec playwright test \
--config=test/e2e-playwright/playwright.config.ts \
--project=packaged-gfs-upload-v2 test/e2e-playwright/gfs-upload-v2.test.ts
fi
if [[ "$run_integration" == "1" ]]; then
: "${CONTROL_API_REAL_PG_ADMIN_URL:?Set CONTROL_API_REAL_PG_ADMIN_URL for the real-Postgres integration lane}"
- npm --prefix control-api test -- --run \
- test/gfsUploadSession.realPostgres.integration.test.ts
+ real_pg_log="$(mktemp)"
+ real_pg_normalized="$(mktemp)"
+ trap 'rm -f "$real_pg_log" "$real_pg_normalized"' EXIT
+ npm --prefix control-api test -- --run realPostgres 2>&1 | tee "$real_pg_log"
+ sed $'s/\033\\[[0-9;]*m//g' "$real_pg_log" >"$real_pg_normalized"
+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$real_pg_normalized" || {
+ echo 'real-Postgres E2E lane executed zero passing test files' >&2
+ exit 1
+ }
+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$real_pg_normalized" || {
+ echo 'real-Postgres E2E lane executed zero passing tests' >&2
+ exit 1
+ }
# Runtime restart/chaos/load is caller-owned because it must be bound to the
# exact branch profile and Cloudflare-backed dev hostname. Do not silently
# substitute an older RED gate or report integration-only evidence as T2.
diff --git a/scripts/tests/test-gfs-upload-v2-geometry-contract.sh b/scripts/tests/test-gfs-upload-v2-geometry-contract.sh
new file mode 100755
index 000000000..b76ba3223
--- /dev/null
+++ b/scripts/tests/test-gfs-upload-v2-geometry-contract.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT"
+
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+read_const() {
+ local file="$1" name="$2"
+ sed -nE "s/^(export )?const ${name} = (.*);?$/\\2/p" "$file" \
+ | head -1 \
+ | tr -d '[:space:]'
+}
+
+gfsc_preferred="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"
+gfsc_max="$(read_const gfs-controller/src/upload/protocol.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"
+ui_preferred="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_PREFERRED_PART_BYTES)"
+ui_max="$(read_const control-ui/app/constants/gfsFileUpload.ts GFS_FILE_UPLOAD_MAX_PART_BYTES)"
+desktop_preferred="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_PREFERRED_PART_BYTES)"
+desktop_max="$(read_const desktop-app/src/gfs/upload.ts GFS_UPLOAD_V2_MAX_PART_BYTES)"
+
+[[ -n "$gfsc_preferred" && "$gfsc_preferred" == "$ui_preferred" && "$gfsc_preferred" == "$desktop_preferred" ]] \
+ || fail "preferred part geometry diverges between writer, Control UI, and Desktop"
+[[ -n "$gfsc_max" && "$gfsc_max" == "$ui_max" && "$gfsc_max" == "$desktop_max" ]] \
+ || fail "maximum part geometry diverges between writer, Control UI, and Desktop"
+
+echo "PASS: GFS Upload v2 part geometry is aligned across writer, Control UI, and Desktop"
diff --git a/tests/e2e/gfsUploadV2Fixtures.ts b/tests/e2e/gfsUploadV2Fixtures.ts
index 170cd6f99..68add8a36 100644
--- a/tests/e2e/gfsUploadV2Fixtures.ts
+++ b/tests/e2e/gfsUploadV2Fixtures.ts
@@ -44,6 +44,30 @@ export async function createDiskUploadFixture(
}
}
+/**
+ * Creates the single-byte-over-limit fixture used by the opt-in negative
+ * browser journey. Keep the normal fixture helper fail-closed at the product
+ * ceiling so a positive test can never accidentally become an oversize test.
+ */
+export async function createOversizedDiskUploadFixture(
+ extension = '.parquet',
+ label = 'gfs-v2-oversize'
+): Promise<DiskUploadFixture> {
+ const byteLength = GFS_UPLOAD_V2_MAX_BYTES + 1
+ const directory = await mkdtemp(path.join(os.tmpdir(), 'evenfire-gfs-upload-v2-'))
+ const fileName = `${label}-${byteLength}${extension}`
+ const filePath = path.join(directory, fileName)
+ await writeFile(filePath, Buffer.alloc(0))
+ await truncate(filePath, byteLength)
+ return {
+ directory,
+ filePath,
+ fileName,
+ byteLength,
+ sha256: await sha256File(filePath),
+ }
+}
+
export async function sha256File(filePath: string): Promise<string> {
const hash = createHash('sha256')
for await (const chunk of createReadStream(filePath)) hash.update(chunk as Buffer)
diff --git a/tests/e2e/gfsUploadV2Runtime.ts b/tests/e2e/gfsUploadV2Runtime.ts
new file mode 100644
index 000000000..86ae738e8
--- /dev/null
+++ b/tests/e2e/gfsUploadV2Runtime.ts
@@ -0,0 +1,342 @@
+import { execFile, execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { resolve } from 'node:path'
+import { promisify } from 'node:util'
+import { firstDataLine, kubectlContext, runControlPostgresSql, sqlLiteral } from './gfsFixtureCore'
+
+const execFileAsync = promisify(execFile)
+const HCC_NAMESPACE = 'control-plane'
+const HCC_DEPLOYMENT = 'host-context-controller'
+const GFS_NAMESPACE = 'gfs'
+const GFS_WRITER_DEPLOYMENT = 'gfsc-writer'
+const GFS_READER_DEPLOYMENT = 'gfsc-reader'
+const GFS_V2_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_V2_ENABLED'
+const GFS_V2_WRITER_ENV = 'GFS_UPLOAD_V2_ENABLED'
+const BRANCH_CONTEXT_RE = /^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/
+
+export interface GfsRuntimeImageMarker {
+ clusterFingerprint?: string
+ imageSource?: string
+ imageTag?: string
+ imagesGeneratedAt?: string
+}
+
+export interface ExpectedGfsRuntimeImageMarker {
+ imageSource?: string
+ imageTag?: string
+ clusterFingerprint?: string
+ imagesGeneratedAt?: string
+}
+
+/**
+ * Local Minikube image mode intentionally has no registry tag. GHCR mode must
+ * carry one because it is the immutable image coordinate that proves which
+ * artifact was deployed. Both modes still require the marker's source and
+ * acquisition timestamp so an empty local tag cannot be confused with a
+ * missing marker field.
+ */
+export function validateGfsRuntimeImageMarker(
+ marker: GfsRuntimeImageMarker,
+ expected: ExpectedGfsRuntimeImageMarker = {}
+): void {
+ if (
+ !marker.clusterFingerprint ||
+ !marker.imageSource ||
+ typeof marker.imageTag !== 'string' ||
+ !marker.imagesGeneratedAt
+ ) {
+ throw new Error(
+ 'refusing GFS E2E mutation: pre-gate marker lacks cluster fingerprint, image coordinates, or acquisition stamp'
+ )
+ }
+ if (marker.imageSource !== 'local' && marker.imageSource !== 'ghcr') {
+ throw new Error(`refusing GFS E2E mutation: unsupported imageSource ${marker.imageSource}`)
+ }
+ if (marker.imageSource === 'ghcr' && marker.imageTag.length === 0) {
+ throw new Error(
+ 'refusing GFS E2E mutation: GHCR image marker is missing its immutable imageTag'
+ )
+ }
+ if (expected.imageSource && marker.imageSource !== expected.imageSource) {
+ throw new Error(
+ 'refusing GFS E2E mutation: imageSource does not match the caller-owned profile'
+ )
+ }
+ if (expected.imageTag !== undefined && marker.imageTag !== expected.imageTag) {
+ throw new Error('refusing GFS E2E mutation: imageTag does not match the caller-owned profile')
+ }
+ if (expected.clusterFingerprint && marker.clusterFingerprint !== expected.clusterFingerprint) {
+ throw new Error(
+ 'refusing GFS E2E mutation: clusterFingerprint does not match the caller-owned profile'
+ )
+ }
+ if (expected.imagesGeneratedAt && marker.imagesGeneratedAt !== expected.imagesGeneratedAt) {
+ throw new Error(
+ 'refusing GFS E2E mutation: imagesGeneratedAt does not match the caller-owned profile'
+ )
+ }
+}
+
+function profilePorts(context: string): Map<string, string> {
+ const path =
+ process.env.CLERUM_PROFILE_PORTS_ENV ||
+ process.env.E2E_PROFILE_PORTS_ENV ||
+ resolve(homedir(), '.cache/clerum/minikube-profiles', context, 'ports.env')
+ if (!existsSync(path)) {
+ throw new Error(`refusing GFS E2E mutation: profile ports.env is missing at ${path}`)
+ }
+ const values = new Map<string, string>()
+ for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {
+ const match = /^(?:export\s+)?([A-Z0-9_]+)=(.*)$/.exec(line.trim())
+ if (match) values.set(match[1]!, match[2]!.replace(/^['"]|['"]$/g, ''))
+ }
+ return values
+}
+
+function normalizeUrl(value: string): string {
+ return value.replace(/\/$/, '')
+}
+
+/**
+ * Mutating negative journeys are allowed only on the exact branch profile
+ * whose generated port map is present in the caller environment. A context
+ * name alone is not ownership proof: another healthy Minikube profile could
+ * otherwise receive an HCC mutation or writer restart.
+ */
+async function assertOwnedRuntimeContext(): Promise<void> {
+ const context = kubectlContext()
+ if (/prod|production/i.test(context)) {
+ throw new Error(`refusing GFS E2E mutation on production-like context ${context}`)
+ }
+ const explicitlyAllowed = new Set(
+ (process.env.E2E_ALLOWED_CONTEXTS || '')
+ .split(',')
+ .map(value => value.trim())
+ .filter(Boolean)
+ )
+ if (!BRANCH_CONTEXT_RE.test(context) && !explicitlyAllowed.has(context)) {
+ throw new Error(
+ `refusing GFS E2E mutation on unscoped context ${context}; use a generated branch profile or E2E_ALLOWED_CONTEXTS`
+ )
+ }
+ const ports = profilePorts(context)
+ const required = [
+ ['CONTROL_UI_BASE_URL', 'CONTROL_UI_URL'],
+ ['EXTERNAL_REST_API_BASE_URL', 'EXTERNAL_REST_API_URL'],
+ ] as const
+ for (const [actualKey, profileKey] of required) {
+ const actual = process.env[actualKey] || process.env[actualKey.replace('_BASE_URL', '_URL')]
+ const expected = ports.get(profileKey)
+ if (!expected || !actual || normalizeUrl(actual) !== normalizeUrl(expected)) {
+ throw new Error(
+ `refusing GFS E2E mutation: ${actualKey} does not match ${profileKey} in profile ports.env`
+ )
+ }
+ }
+ const rpcActual = process.env.RPC_PROXY_BASE_URL || process.env.RPC_PROXY_URL
+ if (rpcActual) {
+ const rpcExpected = ports.get('RPC_PROXY_URL')
+ if (!rpcExpected || normalizeUrl(rpcActual) !== normalizeUrl(rpcExpected)) {
+ throw new Error(
+ 'refusing GFS E2E mutation: RPC_PROXY_BASE_URL does not match RPC_PROXY_URL in profile ports.env'
+ )
+ }
+ }
+
+ const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+ cwd: process.cwd(),
+ encoding: 'utf8',
+ }).trim()
+ const dirty = execFileSync('git', ['status', '--porcelain', '--untracked-files=normal'], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ }).trim()
+ if (dirty) {
+ throw new Error(
+ 'refusing GFS E2E mutation: the worktree is dirty; commit and run pre-gate-sync before fault injection'
+ )
+ }
+ const expectedWorktreeId = createHash('sha1').update(repoRoot).digest('hex')
+ const expectedHead = execFileSync('git', ['rev-parse', 'HEAD'], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ }).trim()
+ const marker = JSON.parse(
+ await kubectl([
+ '-n',
+ HCC_NAMESPACE,
+ 'get',
+ 'configmap/clerum-pre-gate-sync-state',
+ '-o',
+ 'json',
+ ])
+ ) as {
+ data?: {
+ worktreeId?: string
+ gitHead?: string
+ clusterFingerprint?: string
+ imageSource?: string
+ imageTag?: string
+ imagesGeneratedAt?: string
+ }
+ }
+ if (marker.data?.worktreeId !== expectedWorktreeId) {
+ throw new Error(
+ 'refusing GFS E2E mutation: cluster ownership marker does not match this worktree; run pre-gate-sync first'
+ )
+ }
+ if (marker.data?.gitHead !== expectedHead) {
+ throw new Error(
+ `refusing GFS E2E mutation: cluster gitHead ${marker.data?.gitHead || '<missing>'} does not match ${expectedHead}`
+ )
+ }
+ const markerData = marker.data
+ const expectedImageSource = process.env.E2E_IMAGE_SOURCE || process.env.IMAGE_SOURCE || 'local'
+ const expectedImageTag = process.env.E2E_IMAGE_TAG || process.env.IMAGE_TAG
+ const expectedClusterFingerprint =
+ process.env.E2E_EXPECTED_CLUSTER_FINGERPRINT || process.env.E2E_CLUSTER_FINGERPRINT
+ const expectedImagesGeneratedAt = process.env.E2E_EXPECTED_IMAGES_GENERATED_AT
+ validateGfsRuntimeImageMarker(markerData ?? {}, {
+ imageSource: expectedImageSource,
+ ...(expectedImageTag !== undefined ? { imageTag: expectedImageTag } : {}),
+ ...(expectedClusterFingerprint ? { clusterFingerprint: expectedClusterFingerprint } : {}),
+ ...(expectedImagesGeneratedAt ? { imagesGeneratedAt: expectedImagesGeneratedAt } : {}),
+ })
+}
+
+async function kubectl(args: string[], timeout = 240_000): Promise<string> {
+ const result = await execFileAsync('kubectl', ['--context', kubectlContext(), ...args], {
+ encoding: 'utf8',
+ timeout,
+ maxBuffer: 2 * 1024 * 1024,
+ })
+ return result.stdout
+}
+
+async function renderedWriterFlag(): Promise<string> {
+ return (
+ await kubectl([
+ '-n',
+ GFS_NAMESPACE,
+ 'get',
+ `deployment/${GFS_WRITER_DEPLOYMENT}`,
+ '-o',
+ `jsonpath={range .spec.template.spec.containers[*]}{range .env[?(@.name=="${GFS_V2_WRITER_ENV}")]}{.value}{end}{end}`,
+ ])
+ ).trim()
+}
+
+async function waitForRenderedWriterFlag(expected: boolean): Promise<void> {
+ const expectedValue = String(expected)
+ const deadline = Date.now() + 240_000
+ let lastValue = ''
+ while (Date.now() < deadline) {
+ lastValue = await renderedWriterFlag()
+ if (lastValue === expectedValue) return
+ await new Promise(resolve => setTimeout(resolve, 1_000))
+ }
+ throw new Error(
+ `GFS Upload v2 runtime flag did not converge to ${expectedValue}; observed ${lastValue || '<empty>'}`
+ )
+}
+
+async function waitForDeployment(deployment: string): Promise<void> {
+ await kubectl([
+ '-n',
+ GFS_NAMESPACE,
+ 'rollout',
+ 'status',
+ `deployment/${deployment}`,
+ '--timeout=240s',
+ ])
+}
+
+/**
+ * Changes the HCC-owned source flag and waits for the generated writer to
+ * converge. The caller must restore the original value in a finally block.
+ */
+export async function setGfsUploadV2Enabled(enabled: boolean): Promise<void> {
+ await assertOwnedRuntimeContext()
+ await kubectl([
+ '-n',
+ HCC_NAMESPACE,
+ 'set',
+ 'env',
+ `deployment/${HCC_DEPLOYMENT}`,
+ `${GFS_V2_HCC_ENV}=${String(enabled)}`,
+ ])
+ await kubectl([
+ '-n',
+ HCC_NAMESPACE,
+ 'rollout',
+ 'status',
+ `deployment/${HCC_DEPLOYMENT}`,
+ '--timeout=240s',
+ ])
+ await waitForRenderedWriterFlag(enabled)
+ await waitForDeployment(GFS_WRITER_DEPLOYMENT)
+ await waitForDeployment(GFS_READER_DEPLOYMENT)
+}
+
+export async function readGfsUploadV2Enabled(): Promise<boolean> {
+ const value = await renderedWriterFlag()
+ if (value !== 'true' && value !== 'false') {
+ throw new Error(
+ `GFS writer rendered an invalid ${GFS_V2_WRITER_ENV} value: ${value || '<empty>'}`
+ )
+ }
+ return value === 'true'
+}
+
+export async function restartGfsWriter(): Promise<void> {
+ await assertOwnedRuntimeContext()
+ // Restart the real writer deployment while the upload is visibly active.
+ // A rollout gives the relay a bounded termination path; deleting the pod
+ // directly can leave an already-streaming PUT socket open until the full
+ // five-minute part timeout, turning this journey into a transport-timeout
+ // test instead of a restart/recovery test.
+ await kubectl(['-n', GFS_NAMESPACE, 'rollout', 'restart', `deployment/${GFS_WRITER_DEPLOYMENT}`])
+ await waitForDeployment(GFS_WRITER_DEPLOYMENT)
+}
+
+/** Revoke only the exact seeded user grant used by the Desktop E2E. */
+export async function revokeGfsUserWriteGrant(resourceId: string, userId: string): Promise<void> {
+ await assertOwnedRuntimeContext()
+ runControlPostgresSql(`
+ DELETE FROM gfs_grants
+ WHERE drive = 'main'
+ AND resource_id = ${sqlLiteral(resourceId)}::uuid
+ AND subject_type = 'user'
+ AND subject_id = ${sqlLiteral(userId)};
+ `)
+}
+
+/**
+ * Legacy fallback must not create an indexed Upload v2 session. Scope the
+ * assertion to the exact seeded parent/name/owner tuple so unrelated active
+ * sessions cannot make the negative journey flaky.
+ */
+export function countGfsCreateSessions(
+ parentResourceId: string,
+ resourceName: string,
+ ownerSubject: string
+): number {
+ const value = firstDataLine(
+ runControlPostgresSql(`
+ SELECT COUNT(*)::int
+ FROM gfs_upload_sessions
+ WHERE drive = 'main'
+ AND owner_subject = ${sqlLiteral(ownerSubject)}
+ AND operation = 'create'
+ AND parent_rid = ${sqlLiteral(parentResourceId)}
+ AND resource_name = ${sqlLiteral(resourceName)};
+ `)
+ )
+ const count = Number(value)
+ if (!Number.isSafeInteger(count) || count < 0) {
+ throw new Error(`invalid GFS v2 session count: ${value || '<empty>'}`)
+ }
+ return count
+}
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 0b79a7bd0..7f2e6a657 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index cdb0121e7..3912175de 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.63",
+ "version": "0.1.64",
"private": true,
"type": "module",
"scripts": {