Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): close upload retry and runtime evidence gaps
Alfredo Lopez3da54c9b34b9
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 596231727..d3423a415 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -182,7 +182,20 @@ jobs:# suites: the DB migration plus the 7 GFS suites (reader role, blob# migration, blob reconciler, audit evidence, copy, readiness, structural# integrity). The matrix `npm test` skips them for lack of the admin URL.- run: npm test -- --run realPostgres+ run: |+ set -euo pipefail+ log_file="$(mktemp)"+ trap 'rm -f "$log_file"' EXIT+ test -n "${CONTROL_API_REAL_PG_ADMIN_URL}"+ npm test -- --run realPostgres 2>&1 | tee "$log_file"+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$log_file" || {+ echo 'real-Postgres control-api lane executed zero passing test files' >&2+ exit 1+ }+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$log_file" || {+ echo 'real-Postgres control-api lane executed zero passing tests' >&2+ exit 1+ }- name: Install gfs-controller dependenciesworking-directory: gfs-controller@@ -194,7 +207,20 @@ jobs:CONTROL_API_REAL_PG_ADMIN_URL: postgresql://[email protected]:15433/postgres# rename-publication concurrency suite — also admin-URL-gated, so the# gfs-controller matrix entry skips it.- run: npm test -- --run realPostgres+ run: |+ set -euo pipefail+ log_file="$(mktemp)"+ trap 'rm -f "$log_file"' EXIT+ test -n "${CONTROL_API_REAL_PG_ADMIN_URL}"+ npm test -- --run realPostgres 2>&1 | tee "$log_file"+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$log_file" || {+ echo 'real-Postgres gfs-controller lane executed zero passing test files' >&2+ exit 1+ }+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$log_file" || {+ echo 'real-Postgres gfs-controller lane executed zero passing tests' >&2+ exit 1+ }# ─── control-ui build + test check ───────────────────────────────────────# control-ui stays out of the SHARED build/test matrix because its tsconfigdiff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.tsindex 0940eb95a..11ce87cea 100644--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts@@ -4,6 +4,9 @@ import {GfsUploadJob,type GfsUploadStatus,assertGfsFileUploadSize,+ isAmbiguousUploadStatus,+ isRetryableUploadStatus,+ normalizeInstabilityFailureThreshold,parseRetryAfter,} from '@lib/gfsFileUpload'@@ -27,6 +30,35 @@ describe('parseRetryAfter', () => {})})+describe('normalizeInstabilityFailureThreshold', () => {+ it('uses the protocol default when older writers omit the field', () => {+ expect(normalizeInstabilityFailureThreshold(undefined)).toBe(3)+ })++ it('accepts bounded writer-configured thresholds and rejects invalid values', () => {+ expect(normalizeInstabilityFailureThreshold(1)).toBe(1)+ expect(normalizeInstabilityFailureThreshold(5)).toBe(5)+ expect(() => normalizeInstabilityFailureThreshold(0)).toThrow(/invalid instability threshold/)+ expect(() => normalizeInstabilityFailureThreshold(101)).toThrow(/invalid instability threshold/)+ expect(() => normalizeInstabilityFailureThreshold(1.5)).toThrow(/invalid instability threshold/)+ })+})++describe('isRetryableUploadStatus', () => {+ it('uses the same transient allowlist as Desktop and does not retry storage exhaustion', () => {+ expect(isRetryableUploadStatus(503)).toBe(true)+ expect(isRetryableUploadStatus(507)).toBe(false)+ expect(isRetryableUploadStatus(501)).toBe(false)+ })++ it('only reconciles statuses whose response may follow a committed write', () => {+ expect(isAmbiguousUploadStatus(500)).toBe(true)+ expect(isAmbiguousUploadStatus(429)).toBe(false)+ expect(isAmbiguousUploadStatus(507)).toBe(false)+ expect(isAmbiguousUploadStatus(501)).toBe(false)+ })+})+class FakeXhr {static requests: FakeXhr[] = []static statuses: number[] = []diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.tsindex a841cbe0c..9fd740f6e 100644--- a/control-ui/lib/gfsFileUpload.ts+++ b/control-ui/lib/gfsFileUpload.ts@@ -9,7 +9,10 @@ import { apiSend } from './api'const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'const GFS_UPLOAD_DRIVE = 'main'-const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])+export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])+const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])+export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3+const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250@@ -17,6 +20,14 @@ const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60_000const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3+export function isRetryableUploadStatus(status: number): boolean {+ return GFS_UPLOAD_RETRYABLE_STATUS.has(status)+}++export function isAmbiguousUploadStatus(status: number): boolean {+ return GFS_UPLOAD_AMBIGUOUS_STATUS.has(status)+}+export interface GfsUploadTarget {operation: 'create' | 'replace'parentRid?: string@@ -129,6 +140,20 @@ export class GfsUploadCapabilityError extends Error {}}+export function normalizeInstabilityFailureThreshold(value: unknown): number {+ if (value === undefined) return GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD+ if (+ !Number.isSafeInteger(value) ||+ (value as number) < 1 ||+ (value as number) > GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD+ ) {+ throw new GfsUploadCapabilityError(+ 'Resumable GFS upload capabilities advertised an invalid instability threshold.'+ )+ }+ return value as number+}+export const GFS_LEGACY_UPLOAD_MAX_BYTES = 16 * 1024 * 1024function uuid(): string {@@ -263,7 +288,7 @@ function assertRetryable(error: unknown): boolean {return falseconst status =typeof error === 'object' && error !== null ? (error as { status?: unknown }).status : undefined- return typeof status === 'number' ? RETRYABLE_STATUS.has(status) : true+ return typeof status === 'number' ? isRetryableUploadStatus(status) : true}function delay(ms: number, signal?: AbortSignal): Promise<void> {@@ -306,7 +331,7 @@ function ambiguousLifecycleError(error: unknown): boolean {// Transport failures have no HTTP status; keep them ambiguous so completion// reconciles durable state instead of assuming that the request was lost.if (typeof status !== 'number') return true- return status === 408 || status >= 500+ return isAmbiguousUploadStatus(status)}function terminalReconciliationError(error: unknown): boolean {@@ -489,7 +514,7 @@ async function loadUploadStatus(}function ambiguousPartResponse(status: number | undefined): boolean {- return status === undefined || status === 408 || status >= 500+ return status === undefined || isAmbiguousUploadStatus(status)}function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {@@ -683,7 +708,8 @@ async function runParts(onProgress?: (progress: GfsUploadProgress) => void,onByteProgress?: (progress: GfsUploadProgress) => void,onPartFailure?: (partNumber: number) => void,- fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY+ fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY,+ instabilityFailureThreshold = GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD): Promise<{ failed: typeof parts; retryableFailures: number; paused: boolean }> {const initialConcurrency = Math.max(1, Math.min(concurrency, parts.length || 1))const effectiveFallback = Math.max(@@ -703,7 +729,11 @@ async function runParts(const registerRetryableFailure = (): void => {retryableFailures += 1consecutiveRetryableFailures += 1- if (!downgraded && consecutiveRetryableFailures >= 3 && initialConcurrency > effectiveFallback)+ if (+ !downgraded &&+ consecutiveRetryableFailures >= instabilityFailureThreshold &&+ initialConcurrency > effectiveFallback+ )downgraded = true}const registerPartSuccess = (): void => {@@ -1225,6 +1255,9 @@ export class GfsUploadJob {: GFS_FILE_UPLOAD_DEFAULT_CONCURRENCYconst fallbackConcurrency =Number(prepared.resumable.fallbackConcurrency) || GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY+ const instabilityFailureThreshold = normalizeInstabilityFailureThreshold(+ prepared.resumable.instabilityFailureThreshold+ )const result = await runParts(prepared.session,this.input.file,@@ -1238,7 +1271,8 @@ export class GfsUploadJob {inFlightLoaded.delete(partNumber)this.uploadedBytes = calculateUploadedBytes()},- fallbackConcurrency+ fallbackConcurrency,+ instabilityFailureThreshold)if (result.paused || this.pauseRequested) {this.state = 'paused'diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.tsindex 8bf873a08..341ba7d5c 100644--- a/desktop-app/src/gfs/upload.test.ts+++ b/desktop-app/src/gfs/upload.test.ts@@ -3,7 +3,13 @@ import { createHash } from 'node:crypto'import { mkdtemp, rm, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'-import { DesktopGfsUploadJob, uploadLocalFile } from './upload.js'+import {+ DesktopGfsUploadJob,+ isAmbiguousUploadStatus,+ isRetryableUploadStatus,+ normalizeInstabilityFailureThreshold,+ uploadLocalFile,+} from './upload.js'function requestPath(url: string): string {return new URL(url).pathname@@ -18,6 +24,23 @@ function enabledCapabilities(overrides: Record<string, unknown> = {}) {}describe('desktop GFS indexed uploader', () => {+ it('uses the writer threshold and rejects invalid capability values', () => {+ expect(normalizeInstabilityFailureThreshold(undefined)).toBe(3)+ expect(normalizeInstabilityFailureThreshold(1)).toBe(1)+ expect(normalizeInstabilityFailureThreshold(5)).toBe(5)+ expect(() => normalizeInstabilityFailureThreshold(0)).toThrow(/invalid instability threshold/)+ expect(() => normalizeInstabilityFailureThreshold(101)).toThrow(/invalid instability threshold/)+ })++ it('uses the same transient status allowlist as Control UI', () => {+ expect(isRetryableUploadStatus(503)).toBe(true)+ expect(isRetryableUploadStatus(507)).toBe(false)+ expect(isRetryableUploadStatus(501)).toBe(false)+ expect(isAmbiguousUploadStatus(500)).toBe(true)+ expect(isAmbiguousUploadStatus(429)).toBe(false)+ expect(isAmbiguousUploadStatus(507)).toBe(false)+ })+it('streams independent binary parts with base64 checksums and completes the session', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-'))try {diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.tsindex 9bc2ed225..1a579e165 100644--- a/desktop-app/src/gfs/upload.ts+++ b/desktop-app/src/gfs/upload.ts@@ -11,6 +11,8 @@ const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3+export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])+const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250@@ -85,6 +87,7 @@ interface UploadCapabilities {preferredChunkBytes?: numbermaxChunkBytes?: numbermaxConcurrentPartsPerSession?: number+ instabilityFailureThreshold?: numberfallbackConcurrency?: number}}@@ -115,6 +118,23 @@ export class DesktopUploadCapabilityError extends Error {}}+export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3+const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100++export function normalizeInstabilityFailureThreshold(value: unknown): number {+ if (value === undefined) return GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD+ if (+ !Number.isSafeInteger(value) ||+ (value as number) < 1 ||+ (value as number) > GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD+ ) {+ throw new DesktopUploadCapabilityError(+ 'Resumable GFS upload capabilities advertised an invalid instability threshold.'+ )+ }+ return value as number+}+export interface DesktopUploadTransport {requestJson<T>(method: 'GET' | 'POST' | 'HEAD' | 'DELETE',@@ -169,6 +189,7 @@ export interface DesktopUploadInput {/** Main-process-only source fence; never crosses the renderer boundary. */sourceIdentity?: DesktopUploadSourceIdentityadvertisedConcurrency?: number+ instabilityFailureThreshold?: numberfallbackConcurrency?: number}@@ -212,8 +233,16 @@ function unwrap<T>(value: unknown): T {return envelope.data}+export function isRetryableUploadStatus(status: number): boolean {+ return GFS_UPLOAD_RETRYABLE_STATUS.has(status)+}++export function isAmbiguousUploadStatus(status: number): boolean {+ return GFS_UPLOAD_AMBIGUOUS_STATUS.has(status)+}+function retryable(status: number): boolean {- return status === 408 || status === 425 || status === 429 || status >= 500+ return isRetryableUploadStatus(status)}function retryableError(error: unknown): boolean {@@ -296,7 +325,7 @@ function ambiguousLifecycleError(error: unknown): boolean {// Transport failures have no HTTP status; keep them ambiguous so completion// reconciles durable state instead of assuming that the request was lost.if (typeof status !== 'number') return true- return status === 408 || status >= 500+ return isAmbiguousUploadStatus(status)}function terminalReconciliationError(error: unknown): boolean {@@ -569,7 +598,7 @@ function ambiguousPartResponse(status: number | undefined): boolean {// A missing response, edge timeout, or proxy/server 5xx may occur after GFSC// committed the part. 425/429 are explicit admission outcomes and can retry// without an extra status round-trip.- return status === undefined || status === 408 || status >= 500+ return status === undefined || isAmbiguousUploadStatus(status)}function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {@@ -800,7 +829,12 @@ async function runParts(const registerRetryableFailure = (): void => {retryableFailures += 1consecutiveRetryableFailures += 1- if (!downgraded && consecutiveRetryableFailures >= 3 && concurrency > fallbackConcurrency)+ if (+ !downgraded &&+ consecutiveRetryableFailures >=+ normalizeInstabilityFailureThreshold(input.instabilityFailureThreshold) &&+ concurrency > fallbackConcurrency+ )downgraded = true}const registerPartSuccess = (): void => {@@ -1286,6 +1320,9 @@ export class DesktopGfsUploadJob {throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)}this.input.advertisedConcurrency = resumable.maxConcurrentPartsPerSession+ this.input.instabilityFailureThreshold = normalizeInstabilityFailureThreshold(+ resumable.instabilityFailureThreshold+ )this.input.fallbackConcurrency = resumable.fallbackConcurrencyconst resumeId = this.session?.uploadId ?? this.input.resumeUploadIdif (resumeId) {diff --git a/docs/architecture/gfs-upload-v2.md b/docs/architecture/gfs-upload-v2.mdindex 624096fe2..7448e4f1f 100644--- a/docs/architecture/gfs-upload-v2.md+++ b/docs/architecture/gfs-upload-v2.md@@ -14,23 +14,32 @@ finalization.This is deliberately not a one-request 200 MiB upload and is not a claim oftus 1.0 compatibility. It does not raise the legacy JSON/base64 body limit.-| Contract | Frozen value |-| --------------------------- | --------------------------------------------------------: |-| Product maximum | `209715200` bytes (200 MiB binary) |-| Oversize boundary | `209715201` is rejected before session/payload allocation |-| Preferred part | `8388608` bytes (8 MiB) |-| Hard part/request maximum | `16777216` bytes (16 MiB) |-| Default session concurrency | 4 parts |-| Instability fallback | 2 parts after 3 consecutive retryable failures/timeouts |-| Legacy raw-file limit | `16777216` bytes (16 MiB), unchanged during rollout |-| Legacy GFSC body cap | `25165824` bytes (24 MiB), unchanged during rollout |-| Protocol arithmetic ceiling | `1073741824` bytes (1 GiB); not an enabled product value |+| Contract | Frozen value |+| --------------------------- | --------------------------------------------------------------------------------------------: |+| Product maximum | `209715200` bytes (200 MiB binary) |+| Oversize boundary | `209715201` is rejected before session/payload allocation |+| Preferred part | `8388608` bytes (8 MiB) |+| Hard part/request maximum | `16777216` bytes (16 MiB) |+| Default session concurrency | 4 parts |+| Instability fallback | 2 parts after the advertised threshold (default 3) of consecutive retryable failures/timeouts |+| Legacy raw-file limit | `16777216` bytes (16 MiB), unchanged during rollout |+| Legacy GFSC body cap | `25165824` bytes (24 MiB), unchanged during rollout |+| Protocol arithmetic ceiling | `1073741824` bytes (1 GiB); not an enabled product value |Cloudflare evaluates the body of each HTTP request. The aggregate file limitis therefore independent of the request limit: every v2 data-bearing requestis at most 16 MiB, and the verified zone setting can force a lower negotiated`maxChunkBytes`. No client may send a single 100 MB or 200 MiB body.+The capability response owns the instability threshold. It is an integer from+1 through 100; omitted legacy responses mean `3`. Both clients use the+advertised value when deciding when to reduce concurrency and reject a malformed+value rather than guessing. The retryable HTTP status allowlist is deliberately+finite and identical in Desktop and Control UI: `408`, `425`, `429`, `500`,+`502`, `503`, and `504`. `507 Insufficient Storage` and every other `5xx` are+terminal until an explicit server-side recovery/retry is requested. A server+may return `Retry-After`; clients honor it within their bounded retry budget.+## Edge admission limitsUpload-v2 has a transport-abuse admission layer before Control API forwards adiff --git a/docs/testing/gfs-upload-v2.md b/docs/testing/gfs-upload-v2.mdindex cde794197..5aace3009 100644--- a/docs/testing/gfs-upload-v2.md+++ b/docs/testing/gfs-upload-v2.md@@ -35,7 +35,9 @@ T0 must execute real tests and fail loudly if zero tests run. It covers:part PUT, pause/resume, complete, and DELETE;- no v2 whole-file `Buffer`/`ArrayBuffer`/base64/JSON/IPC value;- retry classification, response-loss reconciliation, contiguous-vs-committed- progress, four-to-two fallback, and Desktop IPC byte isolation;+ progress, four-to-two fallback using the advertised instability threshold,+ the identical finite retry-status allowlist (including terminal `507`), and+ Desktop IPC byte isolation;- proxy invariants: exact v2 prefix bypasses the broad JSON parser, declaredand observed 16 MiB caps are enforced, `Expect` is stripped, upload headerssurvive, and disconnect aborts upstream.@@ -48,6 +50,9 @@ T0 must execute real tests and fail loudly if zero tests run. It covers:- Desktop state-v2 migration and legacy quarantine, exact owner/team/env/base/drive resume matching, A → logout → B isolation, epoch-stale dispatchrejection, and `suspended_auth` requiring explicit same-scope resume;+- active-part recovery after an unknown commit must decrement+ `active_part_count` exactly once, including when the reconciliation read also+ fails; the durable row state, not the transport outcome, is authoritative;- legacy fallback symlink/path-swap rejection and proof that logout aborts andawaits a pending fallback before clearing the captured credential.@@ -78,6 +83,10 @@ T1 uses real Postgres for migration and role checks. It proves:- same name/size/mtime with different bytes cannot adopt a committed part orcomplete a resumed session.+The dedicated real-Postgres lane is fail-loud: it must report at least one+executed test file and one passed test. A successful job with zero executed or+zero passed tests is a failure, not a skipped/green result.+The 512 MiB/1 GiB arithmetic ceiling is tested only under an explicit protocoltest configuration. It must not become an enabled 200 MiB product capability.diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.tsindex b2d503918..6c7e02a89 100644--- a/gfs-controller/src/config.test.ts+++ b/gfs-controller/src/config.test.ts@@ -12,11 +12,14 @@ describe('GFS_STORAGE_ROLE', () => {expect(loadConfig().storageRole).toBe(role)})- it.each([undefined, ''])('rejects absent or empty role %s instead of granting writer authority', role => {- vi.stubEnv('GFS_DEV_MODE', 'true')- vi.stubEnv('GFS_STORAGE_ROLE', role)- expect(() => loadConfig()).toThrow(/GFS_STORAGE_ROLE must be explicitly set/)- })+ it.each([undefined, ''])(+ 'rejects absent or empty role %s instead of granting writer authority',+ role => {+ vi.stubEnv('GFS_DEV_MODE', 'true')+ vi.stubEnv('GFS_STORAGE_ROLE', role)+ expect(() => loadConfig()).toThrow(/GFS_STORAGE_ROLE must be explicitly set/)+ }+ )it.each(['Writer', 'read', 'writer ', 'unknown'])('rejects unknown value %s', role => {vi.stubEnv('GFS_DEV_MODE', 'true')@@ -73,17 +76,7 @@ describe('GFS_SYNC_COPY_*', () => {'GFS_SYNC_RENAME_MAX_OBJECTS','GFS_SYNC_RENAME_TIMEOUT_MS',])('rejects invalid explicitly configured values for %s', name => {- for (const raw of [- '',- ' ',- '0',- '-1',- '1.5',- 'NaN',- 'Infinity',- '1e3',- '9007199254740992',- ]) {+ for (const raw of ['', ' ', '0', '-1', '1.5', 'NaN', 'Infinity', '1e3', '9007199254740992']) {expect(() => configWith(name, raw), `${name}=${JSON.stringify(raw)}`).toThrow(new RegExp(`${name} must be a positive safe integer`))@@ -165,4 +158,16 @@ describe('GFS_UPLOAD_V2 strict disabled contract', () => {vi.stubEnv('GFS_UPLOAD_FALLBACK_CONCURRENCY', '4')expect(() => loadConfig()).toThrow(/FALLBACK_CONCURRENCY must be lower/)})++ it('accepts the bounded instability threshold and rejects values outside the capability contract', () => {+ vi.stubEnv('GFS_DEV_MODE', 'true')+ vi.stubEnv('GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD', '7')+ expect(loadConfig().uploadV2.instabilityFailureThreshold).toBe(7)+ for (const raw of ['0', '101', '1.5', 'NaN', 'Infinity']) {+ vi.stubEnv('GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD', raw)+ expect(() => loadConfig(), `threshold=${raw}`).toThrow(+ /GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD must be an integer from 1 through 100/+ )+ }+ })})diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex d4698f322..bb561f9ee 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -1,29 +1,44 @@-import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'+import { afterEach, describe, expect, it } from 'vitest'import { createHash } from 'node:crypto'+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'import { tmpdir } from 'node:os'import { join } from 'node:path'import { Readable } from 'node:stream'-import { afterEach, describe, expect, it } from 'vitest'-import { GfsUploadSessionService, type UploadSessionServiceDeps } from './uploadSession'-import { partGeometry } from './protocol'import type { GfsUploadConfig } from '../config'+import {+ CommitOutcomeUnknownError,+ type TransactionOptions,+ type Transactor,+ type TxClient,+} from '../db/writeStore'+import { partGeometry } from './protocol'+import { GfsUploadSessionService, type UploadSessionServiceDeps } from './uploadSession'type Row = Record<string, unknown>class MemoryDb {sessions: Row[] = []parts: Row[] = []+ failReconciliationReads = falseasync query(text: string, values: unknown[] = []): Promise<{ rows: Row[] }> {const sql = text.replace(/\s+/g, ' ').trim()if (sql.startsWith('SELECT pg_advisory_xact_lock')) return { rows: [] }if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE owner_subject')) {const [owner, drive, key] = values.map(String)- return { rows: this.sessions.filter(row => row.owner_subject === owner && row.drive === drive && row.idempotency_key === key) }+ return {+ rows: this.sessions.filter(+ row => row.owner_subject === owner && row.drive === drive && row.idempotency_key === key+ ),+ }}if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE upload_id')) {const [id, drive, owner] = values.map(String)- return { rows: this.sessions.filter(row => row.upload_id === id && row.drive === drive && row.owner_subject === owner) }+ return {+ rows: this.sessions.filter(+ row => row.upload_id === id && row.drive === drive && row.owner_subject === owner+ ),+ }}if (sql.startsWith('SELECT session_epoch, state FROM gfs_upload_sessions WHERE upload_id')) {const [id, drive, owner] = values.map(String)@@ -33,87 +48,210 @@ class MemoryDb {.map(row => ({ session_epoch: row.session_epoch, state: row.state })),}}- if (sql.startsWith('SELECT p.upload_id, p.part_number, p.staging_path FROM gfs_upload_parts p')) {+ if (+ sql.startsWith('SELECT p.upload_id, p.part_number, p.staging_path FROM gfs_upload_parts p')+ ) {return { rows: [] }}- if (sql.startsWith("SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()")) {+ if (+ sql.startsWith(+ "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()"+ )+ ) {return { rows: [] }}if (sql.startsWith('SELECT p.upload_id, p.part_number, p.offset_bytes, p.length_bytes')) {return { rows: [] }}- if (sql.startsWith("SELECT upload_id, state FROM gfs_upload_sessions WHERE state IN ('aborted', 'expired', 'failed', 'completed') AND cleanup_at IS NULL")) {+ if (+ sql.startsWith(+ "SELECT upload_id, state FROM gfs_upload_sessions WHERE state IN ('aborted', 'expired', 'failed', 'completed') AND cleanup_at IS NULL"+ )+ ) {return {rows: this.sessions- .filter(row => ['aborted', 'expired', 'failed', 'completed'].includes(String(row.state)) && row.cleanup_at == null)+ .filter(+ row =>+ ['aborted', 'expired', 'failed', 'completed'].includes(String(row.state)) &&+ row.cleanup_at == null+ ).map(row => ({ upload_id: row.upload_id, state: row.state })),}}- if (sql.startsWith("SELECT upload_id FROM gfs_upload_sessions WHERE state = 'completed' AND cleanup_at <= now()")) {+ if (+ sql.startsWith(+ "SELECT upload_id FROM gfs_upload_sessions WHERE state = 'completed' AND cleanup_at <= now()"+ )+ ) {return { rows: [] }}- if (sql.startsWith('SELECT COALESCE(SUM((expected_bytes - committed_bytes) + expected_bytes)')) {+ if (+ sql.startsWith('SELECT COALESCE(SUM((expected_bytes - committed_bytes) + expected_bytes)')+ ) {const drive = String(values[0])const reserved = this.sessions- .filter(row => row.drive === drive && ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state)))- .reduce((sum, row) => sum + (Number(row.expected_bytes) - Number(row.committed_bytes ?? 0)) + Number(row.expected_bytes), 0)+ .filter(+ row =>+ row.drive === drive &&+ ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state))+ )+ .reduce(+ (sum, row) =>+ sum ++ (Number(row.expected_bytes) - Number(row.committed_bytes ?? 0)) ++ Number(row.expected_bytes),+ 0+ )return { rows: [{ reserved: String(reserved) }] }}- if (sql === "SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'") {- return { rows: [{ count: String(this.sessions.filter(row => row.state === 'finalizing').length) }] }+ if (+ sql === "SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'"+ ) {+ return {+ rows: [{ count: String(this.sessions.filter(row => row.state === 'finalizing').length) }],+ }}if (sql.startsWith('SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions')) {- const [drive, owner] = values.map(value => value === undefined ? undefined : String(value))- const active = this.sessions.filter(row =>- ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state))- && (drive === undefined || row.drive === drive)- && (owner === undefined || row.owner_subject === owner)+ const [drive, owner] = values.map(value => (value === undefined ? undefined : String(value)))+ const active = this.sessions.filter(+ row =>+ ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state)) &&+ (drive === undefined || row.drive === drive) &&+ (owner === undefined || row.owner_subject === owner)).lengthreturn { rows: [{ count: String(active) }] }}if (sql.startsWith('SELECT COUNT(*)::bigint')) {const drive = String(values[0])- return { rows: [{ count: String(this.parts.filter(part => part.state === 'reserved' && this.sessions.some(session => session.upload_id === part.upload_id && session.drive === drive)).length) }] }+ return {+ rows: [+ {+ count: String(+ this.parts.filter(+ part =>+ part.state === 'reserved' &&+ this.sessions.some(+ session => session.upload_id === part.upload_id && session.drive === drive+ )+ ).length+ ),+ },+ ],+ }}if (sql.startsWith('INSERT INTO gfs_upload_sessions')) {- const [uploadId, key, drive, owner, primary, operation, fingerprint, parent, resource, name, ifMatch, expected, partBytes, partCount, sha, expiresAt] = values+ const [+ uploadId,+ key,+ drive,+ owner,+ primary,+ operation,+ fingerprint,+ parent,+ resource,+ name,+ ifMatch,+ expected,+ partBytes,+ partCount,+ sha,+ expiresAt,+ ] = valuesconst row: Row = {- upload_id: String(uploadId), idempotency_key: String(key), drive: String(drive), owner_subject: String(owner), primary_subject: String(primary),- operation: String(operation), request_fingerprint: String(fingerprint), parent_rid: parent, resource_rid: resource,- resource_name: name, if_match: ifMatch, expected_bytes: expected, part_bytes: partBytes, part_count: partCount,- whole_sha256: sha, committed_bytes: 0, contiguous_bytes: 0, committed_part_count: 0, active_part_count: 0,- session_epoch: 0, state: 'initiated', result_resource_id: null, result_version: null, result_sha256: null,- failure_code: null, expires_at: expiresAt, completed_at: null,+ upload_id: String(uploadId),+ idempotency_key: String(key),+ drive: String(drive),+ owner_subject: String(owner),+ primary_subject: String(primary),+ operation: String(operation),+ request_fingerprint: String(fingerprint),+ parent_rid: parent,+ resource_rid: resource,+ resource_name: name,+ if_match: ifMatch,+ expected_bytes: expected,+ part_bytes: partBytes,+ part_count: partCount,+ whole_sha256: sha,+ committed_bytes: 0,+ contiguous_bytes: 0,+ committed_part_count: 0,+ active_part_count: 0,+ session_epoch: 0,+ state: 'initiated',+ result_resource_id: null,+ result_version: null,+ result_sha256: null,+ failure_code: null,+ expires_at: expiresAt,+ completed_at: null,cleanup_at: null,}this.sessions.push(row)return { rows: [row] }}if (sql.startsWith('SELECT upload_id, part_number')) {+ if (+ this.failReconciliationReads &&+ sql.includes('offset_bytes, length_bytes, sha256, state')+ ) {+ throw new Error('synthetic reconciliation read failure')+ }const [uploadId, partNumber] = values- const rows = this.parts.filter(part => part.upload_id === String(uploadId) && (partNumber === undefined || Number(part.part_number) === Number(partNumber)) && (!sql.includes("state = 'committed'") || part.state === 'committed'))+ const rows = this.parts.filter(+ part =>+ part.upload_id === String(uploadId) &&+ (partNumber === undefined || Number(part.part_number) === Number(partNumber)) &&+ (!sql.includes("state = 'committed'") || part.state === 'committed')+ )return { rows }}if (sql.startsWith('INSERT INTO gfs_upload_parts')) {const [uploadId, partNumber, offset, length, sha, staging, epoch] = values- const row: Row = { upload_id: String(uploadId), part_number: partNumber, offset_bytes: offset, length_bytes: length, sha256: sha, state: 'reserved', staging_path: staging, lease_epoch: epoch, committed_at: null }+ const row: Row = {+ upload_id: String(uploadId),+ part_number: partNumber,+ offset_bytes: offset,+ length_bytes: length,+ sha256: sha,+ state: 'reserved',+ staging_path: staging,+ lease_epoch: epoch,+ committed_at: null,+ }this.parts.push(row)return { rows: [] }}if (sql.startsWith('UPDATE gfs_upload_parts SET offset_bytes')) {const [uploadId, partNumber, offset, length, sha, staging, epoch] = values- const row = this.parts.find(part => part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber))!- Object.assign(row, { offset_bytes: offset, length_bytes: length, sha256: sha, state: 'reserved', staging_path: staging, lease_epoch: epoch, committed_at: null })+ const row = this.parts.find(+ part =>+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)+ )!+ Object.assign(row, {+ offset_bytes: offset,+ length_bytes: length,+ sha256: sha,+ state: 'reserved',+ staging_path: staging,+ lease_epoch: epoch,+ committed_at: null,+ })return { rows: [] }}if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'uploading'")) {const row = this.sessions.find(session => session.upload_id === String(values[0]))!- row.state = 'uploading'; row.active_part_count = Number(row.active_part_count) + 1+ row.state = 'uploading'+ row.active_part_count = Number(row.active_part_count) + 1return { rows: [] }}- if (sql.startsWith('UPDATE gfs_upload_parts SET state = \'committed\'')) {+ if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'committed'")) {const [uploadId, partNumber, path] = values- const row = this.parts.find(part => part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber))!+ const row = this.parts.find(+ part =>+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)+ )!Object.assign(row, { state: 'committed', staging_path: path })return { rows: [] }}@@ -125,7 +263,12 @@ class MemoryDb {row.active_part_count = Math.max(0, Number(row.active_part_count) - 1)let contiguous = 0for (let number = 0; number < Number(row.part_count); number += 1) {- const part = this.parts.find(candidate => candidate.upload_id === row.upload_id && Number(candidate.part_number) === number && candidate.state === 'committed')+ const part = this.parts.find(+ candidate =>+ candidate.upload_id === row.upload_id &&+ Number(candidate.part_number) === number &&+ candidate.state === 'committed'+ )if (!part) breakcontiguous += Number(part.length_bytes)}@@ -134,8 +277,15 @@ class MemoryDb {}if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'failed'")) {const uploadId = String(values[0])- for (const row of this.parts.filter(part => part.upload_id === uploadId && part.state === 'reserved')) row.state = 'failed'- return { rows: [] }+ const row = this.parts.find(+ part =>+ part.upload_id === uploadId &&+ part.state === 'reserved' &&+ Number(part.part_number) === Number(values[1])+ )+ if (!row) return { rows: [] }+ row.state = 'failed'+ return { rows: [{ updated: 1 }] }}if (sql.startsWith('UPDATE gfs_upload_sessions SET active_part_count')) {const row = this.sessions.find(session => session.upload_id === String(values[0]))!@@ -154,7 +304,8 @@ class MemoryDb {}if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'aborted'")) {const row = this.sessions.find(session => session.upload_id === String(values[0]))!- row.state = 'aborted'; row.session_epoch = Number(row.session_epoch) + 1+ row.state = 'aborted'+ row.session_epoch = Number(row.session_epoch) + 1return { rows: [] }}if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {@@ -222,11 +373,11 @@ afterEach(async () => {function service(db: MemoryDb,config: GfsUploadConfig = TEST_CONFIG,- overrides: Partial<Pick<UploadSessionServiceDeps, 'now' | 'finalize'>> = {},+ overrides: Partial<Pick<UploadSessionServiceDeps, 'now' | 'finalize' | 'tx'>> = {}): GfsUploadSessionService {return new GfsUploadSessionService({db,- tx: { transaction: async fn => fn(db) },+ tx: overrides.tx ?? { transaction: async fn => fn(db) },blobs: { availableBytes: async () => 10n ** 12n } as never,storageMountPath: tempRoot,config,@@ -234,17 +385,42 @@ function service(})}-async function settleWithin<T>(promise: Promise<T>, timeoutMs = 500): Promise<- | { kind: 'completed'; value: T }- | { kind: 'rejected'; error: unknown }- | { kind: 'timeout' }+class ThrowAfterCommitTransactor implements Transactor {+ private transactionCount = 0++ constructor(+ private readonly delegate: Transactor,+ private readonly afterCommit: () => void+ ) {}++ async transaction<T>(+ fn: (client: TxClient) => Promise<T>,+ options?: TransactionOptions+ ): Promise<T> {+ this.transactionCount += 1+ const result = await this.delegate.transaction(fn, options)+ // create/reservation is the first transaction; the second is the part+ // commit. The callback makes the subsequent reconciliation read fail.+ if (this.transactionCount === 2) {+ this.afterCommit()+ throw new CommitOutcomeUnknownError(new Error('synthetic post-commit response loss'))+ }+ return result+ }+}++async function settleWithin<T>(+ promise: Promise<T>,+ timeoutMs = 500+): Promise<+ { kind: 'completed'; value: T } | { kind: 'rejected'; error: unknown } | { kind: 'timeout' }> {let timer: ReturnType<typeof setTimeout> | undefinedtry {return await Promise.race([promise.then(value => ({ kind: 'completed' as const, value }),- error => ({ kind: 'rejected' as const, error }),+ error => ({ kind: 'rejected' as const, error })),new Promise<{ kind: 'timeout' }>(resolve => {timer = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs)@@ -256,40 +432,152 @@ async function settleWithin<T>(promise: Promise<T>, timeoutMs = 500): Promise<}describe('GfsUploadSessionService', () => {- it('creates idempotent sessions and keeps indexed commits separate from contiguous progress', { timeout: 15_000 }, async () => {+ it(+ 'creates idempotent sessions and keeps indexed commits separate from contiguous progress',+ { timeout: 15_000 },+ async () => {+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))+ const db = new MemoryDb()+ const uploads = service(db)+ const created = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'data.bin',+ sizeBytes: 2097152,+ idempotencyKey: key,+ })+ expect(created.created).toBe(true)+ const replay = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'data.bin',+ sizeBytes: 2097152,+ idempotencyKey: key,+ })+ expect(replay.created).toBe(false)+ expect(replay.session.uploadId).toBe(created.session.uploadId)++ const first = Buffer.alloc(1048576, 0x11)+ const second = Buffer.alloc(1048576, 0x22)+ const firstGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 0)+ const secondGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 1)+ const checksum = (value: Buffer) => createSha(value)+ const outOfOrder = await uploads.putPart(+ created.session.uploadId,+ secondGeometry,+ checksum(second),+ Readable.from([second]) as never,+ principal+ )+ expect(outOfOrder.session.contiguousBytes).toBe(0)+ const ordered = await uploads.putPart(+ created.session.uploadId,+ firstGeometry,+ checksum(first),+ Readable.from([first]) as never,+ principal+ )+ expect(ordered.session.contiguousBytes).toBe(2097152)+ expect(ordered.session.committedPartCount).toBe(2)+ expect(await readFile(ordered.part.stagingPath)).toEqual(first)+ const duplicate = await uploads.putPart(+ created.session.uploadId,+ firstGeometry,+ checksum(first),+ Readable.from([first]) as never,+ principal+ )+ expect(duplicate.part.sha256).toBe(checksum(first))+ }+ )++ it('does not double-decrement active parts after an unknown commit and failed reconciliation', async () => {tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))const db = new MemoryDb()- const uploads = service(db)- const created = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 2097152, idempotencyKey: key })- expect(created.created).toBe(true)- const replay = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 2097152, idempotencyKey: key })- expect(replay.created).toBe(false)- expect(replay.session.uploadId).toBe(created.session.uploadId)+ const created = await service(db).create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'counter-safe.bin',+ sizeBytes: 2 * TEST_CONFIG.preferredPartBytes,+ idempotencyKey: '66666666-6666-4666-8666-666666666666',+ })+ const session = db.sessions[0]!+ session.active_part_count = 1+ db.parts.push({+ upload_id: created.session.uploadId,+ part_number: 1,+ offset_bytes: TEST_CONFIG.preferredPartBytes,+ length_bytes: TEST_CONFIG.preferredPartBytes,+ sha256: 'f'.repeat(64),+ state: 'reserved',+ staging_path: join(tempRoot, 'other.part'),+ lease_epoch: 0,+ committed_at: null,+ })++ const tx = new ThrowAfterCommitTransactor({ transaction: async fn => fn(db) }, () => {+ db.failReconciliationReads = true+ })+ const uploads = service(db, TEST_CONFIG, { tx })+ const first = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x31)+ await expect(+ uploads.putPart(+ created.session.uploadId,+ partGeometry(+ {+ expectedBytes: 2 * TEST_CONFIG.preferredPartBytes,+ partBytes: TEST_CONFIG.preferredPartBytes,+ },+ 0+ ),+ createSha(first),+ Readable.from([first]) as never,+ principal+ )+ ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)- const first = Buffer.alloc(1048576, 0x11)- const second = Buffer.alloc(1048576, 0x22)- const firstGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 0)- const secondGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 1)- const checksum = (value: Buffer) => createSha(value)- const outOfOrder = await uploads.putPart(created.session.uploadId, secondGeometry, checksum(second), Readable.from([second]) as never, principal)- expect(outOfOrder.session.contiguousBytes).toBe(0)- const ordered = await uploads.putPart(created.session.uploadId, firstGeometry, checksum(first), Readable.from([first]) as never, principal)- expect(ordered.session.contiguousBytes).toBe(2097152)- expect(ordered.session.committedPartCount).toBe(2)- expect(await readFile(ordered.part.stagingPath)).toEqual(first)- const duplicate = await uploads.putPart(created.session.uploadId, firstGeometry, checksum(first), Readable.from([first]) as never, principal)- expect(duplicate.part.sha256).toBe(checksum(first))+ expect(session.active_part_count).toBe(1)+ expect(db.parts.find(part => Number(part.part_number) === 0)?.state).toBe('committed')+ expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')})it('cancels terminally and rejects a conflicting idempotency fingerprint', async () => {tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))const db = new MemoryDb()const uploads = service(db)- await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 0, idempotencyKey: key })- await expect(uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'other.bin', sizeBytes: 0, idempotencyKey: key })).rejects.toMatchObject({ code: 'idempotency_conflict' })- const session = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'cancel.bin', sizeBytes: 0, idempotencyKey: '22222222-2222-4222-8222-222222222222' })+ await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'data.bin',+ sizeBytes: 0,+ idempotencyKey: key,+ })+ await expect(+ uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'other.bin',+ sizeBytes: 0,+ idempotencyKey: key,+ })+ ).rejects.toMatchObject({ code: 'idempotency_conflict' })+ const session = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'cancel.bin',+ sizeBytes: 0,+ idempotencyKey: '22222222-2222-4222-8222-222222222222',+ })await uploads.cancel(session.session.uploadId, principal)- await expect(uploads.get(session.session.uploadId, principal)).rejects.toMatchObject({ code: 'upload_aborted' })+ await expect(uploads.get(session.session.uploadId, principal)).rejects.toMatchObject({+ code: 'upload_aborted',+ })})it('releases a rejected finalizer before quiescence and cleans the failed upload immediately', async () => {@@ -302,7 +590,9 @@ describe('GfsUploadSessionService', () => {// five-minute part timeout: the old code still skipped cleanup because// the failed finalizer remained registered while it drained itself.now: () => (now += TEST_CONFIG.partTimeoutMs + 1),- finalize: async () => { throw new Error('synthetic finalizer rejection') },+ finalize: async () => {+ throw new Error('synthetic finalizer rejection')+ },})const created = await uploads.create({...principal,@@ -319,14 +609,17 @@ describe('GfsUploadSessionService', () => {const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))expect(outcome.kind).toBe('rejected')- if (outcome.kind !== 'rejected') throw new Error(`unexpected completion outcome: ${outcome.kind}`)+ if (outcome.kind !== 'rejected')+ throw new Error(`unexpected completion outcome: ${outcome.kind}`)expect(outcome.error).toMatchObject({ message: 'synthetic finalizer rejection' })expect(db.sessions[0]).toMatchObject({state: 'failed',failure_code: 'finalization_failed',cleanup_at: expect.any(String),})- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)+ expect(+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size+ ).toBe(0)expect(db.parts).toHaveLength(0)await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })})@@ -338,9 +631,10 @@ describe('GfsUploadSessionService', () => {const config = { ...TEST_CONFIG, finalizeTimeoutMs: 5 }const uploads = service(db, config, {now: () => (now += config.partTimeoutMs + 1),- finalize: async (_session, _parts, signal) => new Promise((_resolve, reject) => {- signal?.addEventListener('abort', () => reject(signal.reason), { once: true })- }),+ finalize: async (_session, _parts, signal) =>+ new Promise((_resolve, reject) => {+ signal?.addEventListener('abort', () => reject(signal.reason), { once: true })+ }),})const created = await uploads.create({...principal,@@ -357,14 +651,17 @@ describe('GfsUploadSessionService', () => {const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))expect(outcome.kind).toBe('rejected')- if (outcome.kind !== 'rejected') throw new Error(`unexpected completion outcome: ${outcome.kind}`)+ if (outcome.kind !== 'rejected')+ throw new Error(`unexpected completion outcome: ${outcome.kind}`)expect(outcome.error).toMatchObject({ code: 'precondition_failed' })expect(db.sessions[0]).toMatchObject({state: 'failed',failure_code: 'precondition_failed',cleanup_at: expect.any(String),})- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)+ expect(+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size+ ).toBe(0)expect(db.parts).toHaveLength(0)await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })})@@ -375,7 +672,9 @@ describe('GfsUploadSessionService', () => {let now = 0const uploads = service(db, TEST_CONFIG, {now: () => (now += TEST_CONFIG.partTimeoutMs + 1),- finalize: async () => { throw new Error('synthetic cleanup retry') },+ finalize: async () => {+ throw new Error('synthetic cleanup retry')+ },})const created = await uploads.create({...principal,@@ -394,7 +693,9 @@ describe('GfsUploadSessionService', () => {const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))expect(outcome.kind).toBe('rejected')expect(db.sessions[0]).toMatchObject({ state: 'failed', cleanup_at: null })- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)+ expect(+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size+ ).toBe(0)await rm(uploadDirectory, { force: true })await mkdir(uploadDirectory, { recursive: true })diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex 388ccbc1c..e094cfb0a 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -1,26 +1,26 @@import { createHash, randomUUID } from 'node:crypto'import { constants } from 'node:fs'import { lstat, mkdir, open, rename, rm, stat } from 'node:fs/promises'-import { dirname, resolve } from 'node:path'import type { IncomingMessage } from 'node:http'+import { dirname, resolve } from 'node:path'import { Readable, Transform } from 'node:stream'import { pipeline } from 'node:stream/promises'import { GfsError } from '../api/errors'-import { normalizeResourceName, ResourceNameError } from '../api/resourceName'-import { normalizeResourceId, PathError } from '../storage/paths'+import { ResourceNameError, normalizeResourceName } from '../api/resourceName'+import type { GfsUploadConfig } from '../config'import { CommitOutcomeUnknownError, RollbackOutcomeUnknownError } from '../db/writeStore'import type { BlobWriter, Transactor, TxClient } from '../db/writeStore'-import type { GfsUploadConfig } from '../config'+import { PathError, normalizeResourceId } from '../storage/paths'import {GFS_UPLOAD_V2_COMPLETE_BODY_MAX_BYTES,GFS_UPLOAD_V2_METADATA_BODY_MAX_BYTES,GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,+ GfsUploadGeometryError,+ type GfsUploadPartGeometry,disabledGfsUploadV2Capability,partCountFor,partGeometry,validatePartGeometry,- GfsUploadGeometryError,- type GfsUploadPartGeometry,} from './protocol'export type { GfsUploadPartGeometry } from './protocol'@@ -85,9 +85,18 @@ export interface UploadSessionServiceDeps {storageMountPath: stringconfig: GfsUploadConfignow?: () => number- authorize?: (principal: UploadPrincipal, operation: 'create' | 'replace', resourceRid?: string) => Promise<void>+ authorize?: (+ principal: UploadPrincipal,+ operation: 'create' | 'replace',+ resourceRid?: string+ ) => Promise<void>/** PR3 supplies this; PR2 intentionally cannot publish a resource. */- finalize?: (session: UploadSessionRow, parts: UploadPartRow[], signal?: AbortSignal, deadlineAtMs?: number) => Promise<{ resourceId: string; version: number; sha256: string }>+ finalize?: (+ session: UploadSessionRow,+ parts: UploadPartRow[],+ signal?: AbortSignal,+ deadlineAtMs?: number+ ) => Promise<{ resourceId: string; version: number; sha256: string }>}export interface UploadSessionRow {@@ -148,7 +157,8 @@ interface InFlightFinalizer {const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iconst SHA256_RE = /^[0-9a-f]{64}$/i-const TEMP_PART_RE = /^(\d+)\.part\.tmp-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i+const TEMP_PART_RE =+ /^(\d+)\.part\.tmp-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/ifunction rowToSession(row: Record<string, unknown>): UploadSessionRow {return {@@ -194,7 +204,11 @@ function rowToPart(row: Record<string, unknown>): UploadPartRow {}}-function canonicalFingerprint(input: CreateUploadSessionInput, partBytes: number, partCount: number): string {+function canonicalFingerprint(+ input: CreateUploadSessionInput,+ partBytes: number,+ partCount: number+): string {return JSON.stringify({owner: input.ownerSubject,drive: input.drive,@@ -236,7 +250,8 @@ function requireUuid(value: string, field: string): string {}function requireChecksum(value: string | undefined, field: string): string {- if (value === undefined || !SHA256_RE.test(value)) throw new GfsError('path_invalid', `${field} must be a SHA-256 hex digest`)+ if (value === undefined || !SHA256_RE.test(value))+ throw new GfsError('path_invalid', `${field} must be a SHA-256 hex digest`)return value.toLowerCase()}@@ -265,7 +280,10 @@ function toReceipt(session: UploadSessionRow): UploadSessionReceipt {}function assertNotExpired(session: UploadSessionRow, now: number): void {- if (Date.parse(session.expiresAt) <= now && !['completed', 'aborted', 'expired'].includes(session.state)) {+ if (+ Date.parse(session.expiresAt) <= now &&+ !['completed', 'aborted', 'expired'].includes(session.state)+ ) {throw new GfsError('upload_expired', 'upload session has expired')}}@@ -291,7 +309,12 @@ function sessionPath(root: string, uploadId: string, partNumber: number, suffix:* intentionally rejected because a new lease may own it after the row lock is* released.*/-async function validatedTempPath(root: string, uploadId: string, partNumber: number, candidate: string): Promise<string | null> {+async function validatedTempPath(+ root: string,+ uploadId: string,+ partNumber: number,+ candidate: string+): Promise<string | null> {const canonicalUploadId = requireUuid(uploadId, 'uploadId')const expectedDir = resolve(root, '.uploads', canonicalUploadId, 'parts')const resolved = resolve(candidate)@@ -299,7 +322,12 @@ async function validatedTempPath(root: string, uploadId: string, partNumber: numif (!resolved.startsWith(prefix)) return nullconst match = TEMP_PART_RE.exec(resolved.slice(prefix.length))if (!match || Number(match[1]) !== partNumber) return null- for (const directory of [resolve(root), resolve(root, '.uploads'), resolve(root, '.uploads', canonicalUploadId), expectedDir]) {+ for (const directory of [+ resolve(root),+ resolve(root, '.uploads'),+ resolve(root, '.uploads', canonicalUploadId),+ expectedDir,+ ]) {try {if (!(await lstat(directory)).isDirectory()) return null} catch (error) {@@ -318,7 +346,8 @@ async function ensureStagingDir(path: string): Promise<void> {await mkdir(partsDir, { recursive: true, mode: 0o700 })for (const directory of [root, uploadsDir, uploadDir, partsDir]) {const info = await lstat(directory)- if (!info.isDirectory()) throw new GfsError('path_invalid', 'upload staging ancestors must be real directories')+ if (!info.isDirectory())+ throw new GfsError('path_invalid', 'upload staging ancestors must be real directories')}}@@ -349,7 +378,12 @@ async function removeUploadDirectory(root: string, uploadId: string): Promise<boasync function committedPartFilePresent(root: string, part: UploadPartRow): Promise<boolean> {const expected = sessionPath(root, part.uploadId, part.partNumber, '')if (part.stagingPath !== expected) return false- const safe = await validatedTempPath(root, part.uploadId, part.partNumber, `${expected}.tmp-${randomUUID()}`)+ const safe = await validatedTempPath(+ root,+ part.uploadId,+ part.partNumber,+ `${expected}.tmp-${randomUUID()}`+ )// validatedTempPath also validates all private ancestors. The synthetic// candidate is never unlinked and only serves as the containment check.if (!safe) return false@@ -382,7 +416,11 @@ async function readPartBody(try {if (destination) {await ensureStagingDir(destination)- handle = await open(destination, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600)+ handle = await open(+ destination,+ constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,+ 0o600+ )stream = handle.createWriteStream()}let bytes = 0@@ -413,11 +451,13 @@ async function readPartBody(signal?.throwIfAborted()const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw)bytes += chunk.length- if (bytes > expectedLength || bytes > maxLength) throw new GfsError('payload_too_large', 'part body exceeds its declared length')+ if (bytes > expectedLength || bytes > maxLength)+ throw new GfsError('payload_too_large', 'part body exceeds its declared length')digest.update(chunk)}}- if (bytes !== expectedLength) throw new GfsError('path_invalid', `part body has ${bytes} bytes; expected ${expectedLength}`)+ if (bytes !== expectedLength)+ throw new GfsError('path_invalid', `part body has ${bytes} bytes; expected ${expectedLength}`)if (destination) {const syncHandle = await open(destination, constants.O_RDONLY | constants.O_NOFOLLOW)try {@@ -465,28 +505,47 @@ export class GfsUploadSessionService {}private hasInFlight(uploadId: string): boolean {- return [...this.inFlight.keys()].some(key => key.startsWith(`${uploadId}:`))- || this.inFlightFinalizers.has(uploadId)+ return (+ [...this.inFlight.keys()].some(key => key.startsWith(`${uploadId}:`)) ||+ this.inFlightFinalizers.has(uploadId)+ )}- async create(input: CreateUploadSessionInput): Promise<{ created: boolean; session: UploadSessionReceipt }> {+ async create(+ input: CreateUploadSessionInput+ ): Promise<{ created: boolean; session: UploadSessionReceipt }> {requireEnabled(this.deps.config)const idempotencyKey = requireUuid(input.idempotencyKey, 'idempotencyKey')- if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 0 || input.sizeBytes > GFS_UPLOAD_V2_PRODUCT_MAX_BYTES) {- throw new GfsError('payload_too_large', `sizeBytes must be between 0 and ${GFS_UPLOAD_V2_PRODUCT_MAX_BYTES}`)+ if (+ !Number.isSafeInteger(input.sizeBytes) ||+ input.sizeBytes < 0 ||+ input.sizeBytes > GFS_UPLOAD_V2_PRODUCT_MAX_BYTES+ ) {+ throw new GfsError(+ 'payload_too_large',+ `sizeBytes must be between 0 and ${GFS_UPLOAD_V2_PRODUCT_MAX_BYTES}`+ )}const name = normalizedName(input.name)const parentRid = normalizedRid(input.parentRid, 'parentRid')const resourceRid = normalizedRid(input.resourceRid, 'resourceRid')- if (input.operation === 'create' && (!parentRid || !name)) throw new GfsError('path_invalid', 'create requires parentRid and name')- if (input.operation === 'replace' && !resourceRid) throw new GfsError('path_invalid', 'replace requires resourceRid')- const wholeSha256 = input.wholeSha256 === null || input.wholeSha256 === undefined- ? null- : requireChecksum(input.wholeSha256, 'wholeSha256')- await this.deps.authorize?.(input, input.operation, input.operation === 'create' ? parentRid : resourceRid)+ if (input.operation === 'create' && (!parentRid || !name))+ throw new GfsError('path_invalid', 'create requires parentRid and name')+ if (input.operation === 'replace' && !resourceRid)+ throw new GfsError('path_invalid', 'replace requires resourceRid')+ const wholeSha256 =+ input.wholeSha256 === null || input.wholeSha256 === undefined+ ? null+ : requireChecksum(input.wholeSha256, 'wholeSha256')+ await this.deps.authorize?.(+ input,+ input.operation,+ input.operation === 'create' ? parentRid : resourceRid+ )const partBytes = this.deps.config.preferredPartBytesconst partCount = partCountFor({ expectedBytes: input.sizeBytes, partBytes })- if (partCount > this.deps.config.maxPartCount) throw new GfsError('payload_too_large', 'upload part count exceeds the configured maximum')+ if (partCount > this.deps.config.maxPartCount)+ throw new GfsError('payload_too_large', 'upload part count exceeds the configured maximum')const canonicalInput = { ...input, name, parentRid, resourceRid, wholeSha256 }const requestFingerprint = canonicalFingerprint(canonicalInput, partBytes, partCount)const existing = await this.deps.db.query(@@ -495,7 +554,11 @@ export class GfsUploadSessionService {)if (existing.rows[0]) {const row = rowToSession(existing.rows[0])- if (row.requestFingerprint !== requestFingerprint) throw new GfsError('idempotency_conflict', 'idempotency key is bound to a different request')+ if (row.requestFingerprint !== requestFingerprint)+ throw new GfsError(+ 'idempotency_conflict',+ 'idempotency key is bound to a different request'+ )return { created: false, session: toReceipt(row) }}const uploadId = requireUuid(input.uploadId ?? randomUUID(), 'uploadId')@@ -503,7 +566,10 @@ export class GfsUploadSessionService {const available = this.deps.blobs.availableBytes ? await this.deps.blobs.availableBytes() : nulltry {const result = await this.deps.tx.transaction(async client => {- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:admission:' || $1)::bigint)`, [input.drive])+ await client.query(+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:admission:' || $1)::bigint)`,+ [input.drive]+ )const subjectActive = await client.query(`SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessionsWHERE drive = $1 AND owner_subject = $2 AND state IN ('initiated','uploading','paused','finalizing') AND expires_at > now()`,@@ -528,8 +594,14 @@ export class GfsUploadSessionService {[input.drive])const reservedBytes = BigInt(String(reserved.rows[0]?.reserved ?? 0))- if (reservedBytes + BigInt(input.sizeBytes) * 2n + BigInt(this.deps.config.minFreeBytes) > available) {- throw new GfsError('insufficient_storage', 'insufficient storage for upload reservation')+ if (+ reservedBytes + BigInt(input.sizeBytes) * 2n + BigInt(this.deps.config.minFreeBytes) >+ available+ ) {+ throw new GfsError(+ 'insufficient_storage',+ 'insufficient storage for upload reservation'+ )}}const inserted = await client.query(@@ -540,10 +612,24 @@ export class GfsUploadSessionService {whole_sha256, state, expires_at)VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,'initiated',$16)RETURNING *`,- [uploadId, idempotencyKey, input.drive, input.ownerSubject, input.primarySubject,- input.operation, requestFingerprint, parentRid ?? null, resourceRid ?? null,- name ?? null, input.ifMatch ?? null, input.sizeBytes, partBytes, partCount,- wholeSha256, expiresAt]+ [+ uploadId,+ idempotencyKey,+ input.drive,+ input.ownerSubject,+ input.primarySubject,+ input.operation,+ requestFingerprint,+ parentRid ?? null,+ resourceRid ?? null,+ name ?? null,+ input.ifMatch ?? null,+ input.sizeBytes,+ partBytes,+ partCount,+ wholeSha256,+ expiresAt,+ ])return rowToSession(inserted.rows[0]!)})@@ -556,7 +642,11 @@ export class GfsUploadSessionService {)const row = replay.rows[0] ? rowToSession(replay.rows[0]) : nullif (!row) throw error- if (row.requestFingerprint !== requestFingerprint) throw new GfsError('idempotency_conflict', 'idempotency key is bound to a different request')+ if (row.requestFingerprint !== requestFingerprint)+ throw new GfsError(+ 'idempotency_conflict',+ 'idempotency key is bound to a different request'+ )return { created: false, session: toReceipt(row) }}}@@ -584,14 +674,20 @@ export class GfsUploadSessionService {}}- async status(uploadId: string, principal: UploadPrincipal, options?: { cursor?: string; limit?: number }): Promise<UploadStatusPage> {+ async status(+ uploadId: string,+ principal: UploadPrincipal,+ options?: { cursor?: string; limit?: number }+ ): Promise<UploadStatusPage> {requireEnabled(this.deps.config)const session = await this.loadOwned(uploadId, principal)const rawCursor = options?.cursor ?? ''- if (rawCursor && !/^\d+$/.test(rawCursor)) throw new GfsError('path_invalid', 'status cursor is invalid')+ if (rawCursor && !/^\d+$/.test(rawCursor))+ throw new GfsError('path_invalid', 'status cursor is invalid')const cursor = rawCursor ? Number(rawCursor) : -1const requestedLimit = options?.limit ?? 256- if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1 || requestedLimit > 256) throw new GfsError('path_invalid', 'status limit is invalid')+ if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1 || requestedLimit > 256)+ throw new GfsError('path_invalid', 'status limit is invalid')const result = await this.deps.db.query(`SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epochFROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' AND part_number > $2 ORDER BY part_number ASC LIMIT $3`,@@ -625,14 +721,26 @@ export class GfsUploadSessionService {requireEnabled(this.deps.config)const expectedChecksum = requireChecksum(checksum, 'Upload-Checksum')const session = await this.loadOwned(uploadId, principal)- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')+ await this.deps.authorize?.(+ principal,+ session.operation,+ session.operation === 'create'+ ? (session.parentRid ?? undefined)+ : (session.resourceRid ?? undefined)+ )+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is pausing')assertNotExpired(session, this.now())if (session.state === 'paused') throw new GfsError('conflict', 'upload session is paused')- if (['finalizing', 'completed'].includes(session.state)) throw new GfsError('upload_completed', 'upload session is no longer writable')- if (['aborted', 'expired', 'failed'].includes(session.state)) throw new GfsError('upload_aborted', 'upload session is not writable')+ if (['finalizing', 'completed'].includes(session.state))+ throw new GfsError('upload_completed', 'upload session is no longer writable')+ if (['aborted', 'expired', 'failed'].includes(session.state))+ throw new GfsError('upload_aborted', 'upload session is not writable')try {- validatePartGeometry({ expectedBytes: session.expectedBytes, partBytes: session.partBytes }, part)+ validatePartGeometry(+ { expectedBytes: session.expectedBytes, partBytes: session.partBytes },+ part+ )} catch (error) {if (error instanceof GfsUploadGeometryError) throw new GfsError('path_invalid', error.message)throw error@@ -653,56 +761,94 @@ export class GfsUploadSessionService {}try {reserved = await this.deps.tx.transaction(async client => {- const locked = await this.lockSession(client, uploadId, principal)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')- if (locked.state === 'paused') throw new GfsError('conflict', 'upload session is paused')- if (locked.state !== 'initiated' && locked.state !== 'uploading') throw new GfsError('upload_aborted', 'upload session is not writable')- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:streams:' || $1)::bigint)`, [locked.drive])- const existing = await client.query(- `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch+ const locked = await this.lockSession(client, uploadId, principal)+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is pausing')+ if (locked.state === 'paused') throw new GfsError('conflict', 'upload session is paused')+ if (locked.state !== 'initiated' && locked.state !== 'uploading')+ throw new GfsError('upload_aborted', 'upload session is not writable')+ await client.query(+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:streams:' || $1)::bigint)`,+ [locked.drive]+ )+ const existing = await client.query(+ `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epochFROM gfs_upload_parts WHERE upload_id = $1 AND part_number = $2 FOR UPDATE`,- [uploadId, part.partNumber]- )- if (existing.rows[0] && String(existing.rows[0].state) === 'committed') return { session: locked, existing: rowToPart(existing.rows[0]) }- if (existing.rows[0] && String(existing.rows[0].state) === 'reserved') {- throw new GfsError('conflict', 'upload part is already in flight')- }- const activeGlobal = await client.query(- `SELECT COUNT(*)::bigint AS count+ [uploadId, part.partNumber]+ )+ if (existing.rows[0] && String(existing.rows[0].state) === 'committed')+ return { session: locked, existing: rowToPart(existing.rows[0]) }+ if (existing.rows[0] && String(existing.rows[0].state) === 'reserved') {+ throw new GfsError('conflict', 'upload part is already in flight')+ }+ const activeGlobal = await client.query(+ `SELECT COUNT(*)::bigint AS countFROM gfs_upload_parts pJOIN gfs_upload_sessions s ON s.upload_id = p.upload_idWHERE s.drive = $1 AND p.state = 'reserved'`,- [locked.drive]- )- if (Number(activeGlobal.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentPartStreamsGlobal) {- throw new GfsError('quota_exceeded', 'global upload stream concurrency limit reached')- }- if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession) throw new GfsError('quota_exceeded', 'per-session part concurrency limit reached')- const stagingPath = sessionPath(this.deps.storageMountPath, uploadId, part.partNumber, `.tmp-${randomUUID()}`)- const previousLeaseEpoch = existing.rows[0] ? Number(existing.rows[0].lease_epoch) : locked.sessionEpoch- const leaseEpoch = Math.max(locked.sessionEpoch + 1, previousLeaseEpoch + 1)- if (existing.rows[0]) {- await client.query(- `UPDATE gfs_upload_parts+ [locked.drive]+ )+ if (+ Number(activeGlobal.rows[0]?.count ?? 0) >=+ this.deps.config.maxConcurrentPartStreamsGlobal+ ) {+ throw new GfsError('quota_exceeded', 'global upload stream concurrency limit reached')+ }+ if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession)+ throw new GfsError('quota_exceeded', 'per-session part concurrency limit reached')+ const stagingPath = sessionPath(+ this.deps.storageMountPath,+ uploadId,+ part.partNumber,+ `.tmp-${randomUUID()}`+ )+ const previousLeaseEpoch = existing.rows[0]+ ? Number(existing.rows[0].lease_epoch)+ : locked.sessionEpoch+ const leaseEpoch = Math.max(locked.sessionEpoch + 1, previousLeaseEpoch + 1)+ if (existing.rows[0]) {+ await client.query(+ `UPDATE gfs_upload_partsSET offset_bytes = $3, length_bytes = $4, sha256 = $5, state = 'reserved',staging_path = $6, lease_epoch = $7, lease_started_at = now(), committed_at = NULLWHERE upload_id = $1 AND part_number = $2`,- [uploadId, part.partNumber, part.offsetBytes, part.lengthBytes, expectedChecksum, stagingPath, leaseEpoch]- )- } else {- await client.query(- `INSERT INTO gfs_upload_parts+ [+ uploadId,+ part.partNumber,+ part.offsetBytes,+ part.lengthBytes,+ expectedChecksum,+ stagingPath,+ leaseEpoch,+ ]+ )+ } else {+ await client.query(+ `INSERT INTO gfs_upload_parts(upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch, lease_started_at)VALUES ($1,$2,$3,$4,$5,'reserved',$6,$7,now())`,- [uploadId, part.partNumber, part.offsetBytes, part.lengthBytes, expectedChecksum, stagingPath, leaseEpoch]- )- }- await client.query(- `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = active_part_count + 1, updated_at = now()+ [+ uploadId,+ part.partNumber,+ part.offsetBytes,+ part.lengthBytes,+ expectedChecksum,+ stagingPath,+ leaseEpoch,+ ]+ )+ }+ await client.query(+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = active_part_count + 1, updated_at = now()WHERE upload_id = $1`,- [uploadId]- )- return { session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 }, existing: null as UploadPartRow | null, stagingPath, leaseEpoch }+ [uploadId]+ )+ return {+ session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 },+ existing: null as UploadPartRow | null,+ stagingPath,+ leaseEpoch,+ }})} catch (error) {this.inFlight.delete(key)@@ -723,18 +869,31 @@ export class GfsUploadSessionService {)const postReplayRow = postReplay.rows[0]if (- this.canceledUploads.has(uploadId)- || !postReplayRow- || String(postReplayRow.state) === 'aborted'- || Number(postReplayRow.session_epoch) !== reserved.session.sessionEpoch+ this.canceledUploads.has(uploadId) ||+ !postReplayRow ||+ String(postReplayRow.state) === 'aborted' ||+ Number(postReplayRow.session_epoch) !== reserved.session.sessionEpoch) {- controller.abort(new GfsError('upload_aborted', 'upload session changed while replay was starting'))+ controller.abort(+ new GfsError('upload_aborted', 'upload session changed while replay was starting')+ )}- const observed = await readPartBody(req, reserved.existing.lengthBytes, this.deps.config.maxPartBytes, undefined, controller.signal)- if (observed.sha256 !== reserved.existing.sha256 || observed.bytes !== reserved.existing.lengthBytes || expectedChecksum !== reserved.existing.sha256) {+ const observed = await readPartBody(+ req,+ reserved.existing.lengthBytes,+ this.deps.config.maxPartBytes,+ undefined,+ controller.signal+ )+ if (+ observed.sha256 !== reserved.existing.sha256 ||+ observed.bytes !== reserved.existing.lengthBytes ||+ expectedChecksum !== reserved.existing.sha256+ ) {throw new GfsError('conflict', 'committed part checksum or length differs')}- if (this.canceledUploads.has(uploadId)) throw new GfsError('upload_aborted', 'upload session was canceled')+ if (this.canceledUploads.has(uploadId))+ throw new GfsError('upload_aborted', 'upload session was canceled')return { part: reserved.existing, session: toReceipt(reserved.session) }} finally {clearTimeout(replayTimer)@@ -756,15 +915,24 @@ export class GfsUploadSessionService {)const postReservationRow = postReservation.rows[0]if (- this.canceledUploads.has(uploadId)- || !postReservationRow- || String(postReservationRow.state) === 'aborted'- || Number(postReservationRow.session_epoch) !== reserved.session.sessionEpoch+ this.canceledUploads.has(uploadId) ||+ !postReservationRow ||+ String(postReservationRow.state) === 'aborted' ||+ Number(postReservationRow.session_epoch) !== reserved.session.sessionEpoch) {- controller.abort(new GfsError('upload_aborted', 'upload session changed while part was reserving'))+ controller.abort(+ new GfsError('upload_aborted', 'upload session changed while part was reserving')+ )}- const observed = await readPartBody(req, part.lengthBytes, this.deps.config.maxPartBytes, stagingPath, controller.signal)- if (observed.sha256 !== expectedChecksum) throw new GfsError('checksum_mismatch', 'part checksum does not match Upload-Checksum')+ const observed = await readPartBody(+ req,+ part.lengthBytes,+ this.deps.config.maxPartBytes,+ stagingPath,+ controller.signal+ )+ if (observed.sha256 !== expectedChecksum)+ throw new GfsError('checksum_mismatch', 'part checksum does not match Upload-Checksum')finalPath = sessionPath(this.deps.storageMountPath, uploadId, part.partNumber, '')try {await stat(finalPath)@@ -782,7 +950,13 @@ export class GfsUploadSessionService {[uploadId, part.partNumber])const row = rowResult.rows[0] ? rowToPart(rowResult.rows[0]) : null- if (!row || row.state !== 'reserved' || row.leaseEpoch !== reserved.leaseEpoch || locked.sessionEpoch !== reserved.session.sessionEpoch || locked.state !== 'uploading') {+ if (+ !row ||+ row.state !== 'reserved' ||+ row.leaseEpoch !== reserved.leaseEpoch ||+ locked.sessionEpoch !== reserved.session.sessionEpoch ||+ locked.state !== 'uploading'+ ) {throw new GfsError('upload_aborted', 'upload session changed while part was in flight')}await client.query(@@ -836,21 +1010,31 @@ export class GfsUploadSessionService {reconciliation = undefined}}- if (reconciliation) return { part: reconciliation.part, session: toReceipt(reconciliation.session) }+ if (reconciliation)+ return { part: reconciliation.part, session: toReceipt(reconciliation.session) }await removeIfPresent(stagingPath)if (finalPath && reconciliation !== undefined) await removeIfPresent(finalPath)- await this.deps.tx.transaction(async client => {- await client.query(- `UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL- WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'`,- [uploadId, part.partNumber]- )- await client.query(- `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()- WHERE upload_id = $1`,- [uploadId]- )- }).catch(() => undefined)+ await this.deps.tx+ .transaction(async client => {+ const failedPart = await client.query(+ `UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL+ WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'+ RETURNING 1`,+ [uploadId, part.partNumber]+ )+ // The commit transaction already decremented the durable counter when+ // it reached PostgreSQL. Only a part that was still reserved needs a+ // recovery decrement; otherwise a lost response plus an inconclusive+ // reconciliation would undercount other concurrent parts.+ if (failedPart.rows.length > 0) {+ await client.query(+ `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()+ WHERE upload_id = $1`,+ [uploadId]+ )+ }+ })+ .catch(() => undefined)throw error} finally {clearTimeout(partTimer)@@ -861,10 +1045,18 @@ export class GfsUploadSessionService {async pause(uploadId: string, principal: UploadPrincipal): Promise<UploadSessionReceipt> {requireEnabled(this.deps.config)const session = await this.loadOwned(uploadId, principal)- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)+ await this.deps.authorize?.(+ principal,+ session.operation,+ session.operation === 'create'+ ? (session.parentRid ?? undefined)+ : (session.resourceRid ?? undefined)+ )if (session.state === 'paused') return toReceipt(session)- if (session.state !== 'initiated' && session.state !== 'uploading') throw new GfsError('conflict', `cannot pause upload in ${session.state}`)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is already pausing')+ if (session.state !== 'initiated' && session.state !== 'uploading')+ throw new GfsError('conflict', `cannot pause upload in ${session.state}`)+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is already pausing')this.pauseRequested.add(uploadId)try {const deadline = this.now() + this.deps.config.partTimeoutMs@@ -883,13 +1075,22 @@ export class GfsUploadSessionService {throw new GfsError('conflict', `cannot pause upload in ${locked.state}`)}if (locked.activePartCount !== 0) return locked- await client.query(`UPDATE gfs_upload_sessions SET state = 'paused', updated_at = now() WHERE upload_id = $1 AND state IN ('initiated','uploading') AND active_part_count = 0`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET state = 'paused', updated_at = now() WHERE upload_id = $1 AND state IN ('initiated','uploading') AND active_part_count = 0`,+ [uploadId]+ )return { ...locked, state: 'paused' }})if (paused.state === 'paused') return toReceipt(paused)}- if (this.now() >= deadline) throw new GfsError('precondition_failed', 'upload pause timed out while draining in-flight parts')- await new Promise<void>(resolve => setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now()))))+ if (this.now() >= deadline)+ throw new GfsError(+ 'precondition_failed',+ 'upload pause timed out while draining in-flight parts'+ )+ await new Promise<void>(resolve =>+ setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now())))+ )}} finally {this.pauseRequested.delete(uploadId)@@ -900,13 +1101,24 @@ export class GfsUploadSessionService {requireEnabled(this.deps.config)const session = await this.loadOwned(uploadId, principal)assertNotExpired(session, this.now())- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')+ await this.deps.authorize?.(+ principal,+ session.operation,+ session.operation === 'create'+ ? (session.parentRid ?? undefined)+ : (session.resourceRid ?? undefined)+ )+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is pausing')if (session.state === 'uploading' || session.state === 'initiated') return toReceipt(session)- if (session.state !== 'paused') throw new GfsError('conflict', `cannot resume upload in ${session.state}`)+ if (session.state !== 'paused')+ throw new GfsError('conflict', `cannot resume upload in ${session.state}`)const resumed = await this.deps.tx.transaction(async client => {const locked = await this.lockSession(client, uploadId, principal)- await client.query(`UPDATE gfs_upload_sessions SET state = 'uploading', updated_at = now() WHERE upload_id = $1 AND state = 'paused'`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET state = 'uploading', updated_at = now() WHERE upload_id = $1 AND state = 'paused'`,+ [uploadId]+ )return { ...locked, state: 'uploading' }})return toReceipt(resumed)@@ -915,30 +1127,47 @@ export class GfsUploadSessionService {async cancel(uploadId: string, principal: UploadPrincipal): Promise<void> {requireEnabled(this.deps.config)const session = await this.loadOwned(uploadId, principal)- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)- if (session.state === 'completed') throw new GfsError('upload_completed', 'completed upload cannot be canceled')- if (session.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is in progress')+ await this.deps.authorize?.(+ principal,+ session.operation,+ session.operation === 'create'+ ? (session.parentRid ?? undefined)+ : (session.resourceRid ?? undefined)+ )+ if (session.state === 'completed')+ throw new GfsError('upload_completed', 'completed upload cannot be canceled')+ if (session.state === 'finalizing')+ throw new GfsError('upload_finalizing', 'upload finalization is in progress')if (session.state === 'aborted') returnthis.canceledUploads.add(uploadId)try {await this.deps.tx.transaction(async client => {const locked = await this.lockSession(client, uploadId, principal)- if (locked.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is in progress')- if (locked.state === 'completed') throw new GfsError('upload_completed', 'completed upload cannot be canceled')+ if (locked.state === 'finalizing')+ throw new GfsError('upload_finalizing', 'upload finalization is in progress')+ if (locked.state === 'completed')+ throw new GfsError('upload_completed', 'completed upload cannot be canceled')// Once the row is fenced as aborted no new reservation can enter. Any// reserved rows are now owned by the cancel drain (or the startup// reconciler), so the durable counter must not keep DELETE waiting for// a process that may have crashed before registering an in-flight// controller.- await client.query(`UPDATE gfs_upload_sessions SET state = 'aborted', session_epoch = session_epoch + 1, active_part_count = 0, updated_at = now() WHERE upload_id = $1`, [uploadId])- await client.query(`UPDATE gfs_upload_parts SET state = 'failed' WHERE upload_id = $1 AND state = 'reserved'`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET state = 'aborted', session_epoch = session_epoch + 1, active_part_count = 0, updated_at = now() WHERE upload_id = $1`,+ [uploadId]+ )+ await client.query(+ `UPDATE gfs_upload_parts SET state = 'failed' WHERE upload_id = $1 AND state = 'reserved'`,+ [uploadId]+ )})} catch (error) {this.canceledUploads.delete(uploadId)throw error}for (const [key, flight] of this.inFlight) {- if (key.startsWith(`${uploadId}:`)) flight.controller.abort(new GfsError('upload_aborted', 'upload canceled'))+ if (key.startsWith(`${uploadId}:`))+ flight.controller.abort(new GfsError('upload_aborted', 'upload canceled'))}await this.waitForActivePartsZero(uploadId, principal)await this.deps.tx.transaction(async client => {@@ -948,7 +1177,10 @@ export class GfsUploadSessionService {if (cleaned) {await this.deps.tx.transaction(async client => {await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'aborted'`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'aborted'`,+ [uploadId]+ )})}this.canceledUploads.delete(uploadId)@@ -975,13 +1207,24 @@ export class GfsUploadSessionService {for (const row of stale.rows) {const uploadId = String(row.upload_id)const partNumber = Number(row.part_number)- const safePath = await validatedTempPath(this.deps.storageMountPath, uploadId, partNumber, String(row.staging_path))+ const safePath = await validatedTempPath(+ this.deps.storageMountPath,+ uploadId,+ partNumber,+ String(row.staging_path)+ )if (!safePath) continue// The row lock fences a new lease for this part. Remove both the// recorded temporary file and a possible rename-before-commit orphan// while the lock is still held; deleting the common final path after// releasing the row would race a fresh reservation.- if (!(await removeStrict(safePath) && await removeStrict(sessionPath(this.deps.storageMountPath, uploadId, partNumber, '')))) continue+ if (+ !(+ (await removeStrict(safePath)) &&+ (await removeStrict(sessionPath(this.deps.storageMountPath, uploadId, partNumber, '')))+ )+ )+ continueconst failed = await client.query(`UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULLWHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'@@ -1065,98 +1308,171 @@ export class GfsUploadSessionService {})for (const [uploadId, cleanup] of uploadDirs) {for (const [key, flight] of this.inFlight) {- if (key.startsWith(`${uploadId}:`)) flight.controller.abort(new GfsError('upload_aborted', 'upload session expired or was canceled'))+ if (key.startsWith(`${uploadId}:`))+ flight.controller.abort(+ new GfsError('upload_aborted', 'upload session expired or was canceled')+ )}- this.inFlightFinalizers.get(uploadId)?.controller.abort(- new GfsError('upload_aborted', 'upload session expired or was canceled')- )+ this.inFlightFinalizers+ .get(uploadId)+ ?.controller.abort(new GfsError('upload_aborted', 'upload session expired or was canceled'))const deadline = this.now() + this.deps.config.partTimeoutMswhile (this.hasInFlight(uploadId) && this.now() < deadline) {await new Promise<void>(resolve => setTimeout(resolve, 25))}- if (!this.hasInFlight(uploadId) && await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {- await this.deps.tx.transaction(async client => {- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])- if (cleanup.purgeReceipt) {- await client.query(`DELETE FROM gfs_upload_sessions WHERE upload_id = $1 AND state = 'completed'`, [uploadId])- } else {- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed')`, [uploadId])- }- }).catch(() => undefined)+ if (+ !this.hasInFlight(uploadId) &&+ (await removeUploadDirectory(this.deps.storageMountPath, uploadId))+ ) {+ await this.deps.tx+ .transaction(async client => {+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])+ if (cleanup.purgeReceipt) {+ await client.query(+ `DELETE FROM gfs_upload_sessions WHERE upload_id = $1 AND state = 'completed'`,+ [uploadId]+ )+ } else {+ await client.query(+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed')`,+ [uploadId]+ )+ }+ })+ .catch(() => undefined)this.canceledUploads.delete(uploadId)} else {// Rotate failed cleanup attempts so a permanently broken oldest batch// cannot starve newer terminal sessions. The terminal row remains// eligible (`cleanup_at IS NULL`) and will be retried next cycle.- await this.deps.tx.transaction(async client => {- await client.query(- `UPDATE gfs_upload_sessions SET updated_at = now()+ await this.deps.tx+ .transaction(async client => {+ await client.query(+ `UPDATE gfs_upload_sessions SET updated_at = now()WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed') AND cleanup_at IS NULL`,- [uploadId]- )- }).catch(() => undefined)+ [uploadId]+ )+ })+ .catch(() => undefined)}}return { staleParts, expiredSessions }}- async complete(uploadId: string, principal: UploadPrincipal, wholeSha256?: string | null): Promise<UploadSessionReceipt> {+ async complete(+ uploadId: string,+ principal: UploadPrincipal,+ wholeSha256?: string | null+ ): Promise<UploadSessionReceipt> {requireEnabled(this.deps.config)const session = await this.loadOwned(uploadId, principal)assertNotExpired(session, this.now())- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')+ await this.deps.authorize?.(+ principal,+ session.operation,+ session.operation === 'create'+ ? (session.parentRid ?? undefined)+ : (session.resourceRid ?? undefined)+ )+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is pausing')if (session.state === 'completed') return toReceipt(session)- if (session.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is already in progress')- if (!this.deps.finalize) throw new GfsError('not_mounted', 'upload finalization is not available yet')- const expectedWhole = wholeSha256 === null || wholeSha256 === undefined- ? session.wholeSha256- : requireChecksum(wholeSha256, 'wholeSha256')- const started: UploadSessionRow & { parts?: UploadPartRow[] } = await this.deps.tx.transaction(async client => {- const locked = await this.lockSession(client, uploadId, principal)- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')- if (locked.state === 'completed') return locked- if (locked.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is already in progress')- if (!['initiated', 'uploading', 'paused'].includes(locked.state)) {- if (locked.state === 'aborted') throw new GfsError('upload_aborted', 'upload session was canceled')- if (locked.state === 'expired') throw new GfsError('upload_expired', 'upload session has expired')- throw new GfsError('conflict', `cannot complete upload in ${locked.state}`)- }- if (locked.wholeSha256 && expectedWhole && locked.wholeSha256 !== expectedWhole) {- throw new GfsError('conflict', 'whole-file checksum differs from the session checksum')- }- const parts = await client.query(`SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' ORDER BY part_number`, [uploadId])- const committedParts = parts.rows.map(rowToPart)- if (committedParts.length !== locked.partCount) throw new GfsError('upload_incomplete', 'not all upload parts are committed')- for (let index = 0; index < locked.partCount; index += 1) {- const actual = committedParts[index]- const expected = partGeometry({ expectedBytes: locked.expectedBytes, partBytes: locked.partBytes }, index)- if (!actual || actual.partNumber !== expected.partNumber || actual.offsetBytes !== expected.offsetBytes || actual.lengthBytes !== expected.lengthBytes) {- throw new GfsError('upload_incomplete', `upload part ${index} does not match the session geometry`)+ if (session.state === 'finalizing')+ throw new GfsError('upload_finalizing', 'upload finalization is already in progress')+ if (!this.deps.finalize)+ throw new GfsError('not_mounted', 'upload finalization is not available yet')+ const expectedWhole =+ wholeSha256 === null || wholeSha256 === undefined+ ? session.wholeSha256+ : requireChecksum(wholeSha256, 'wholeSha256')+ const started: UploadSessionRow & { parts?: UploadPartRow[] } = await this.deps.tx.transaction(+ async client => {+ const locked = await this.lockSession(client, uploadId, principal)+ if (this.pauseRequested.has(uploadId))+ throw new GfsError('conflict', 'upload session is pausing')+ if (locked.state === 'completed') return locked+ if (locked.state === 'finalizing')+ throw new GfsError('upload_finalizing', 'upload finalization is already in progress')+ if (!['initiated', 'uploading', 'paused'].includes(locked.state)) {+ if (locked.state === 'aborted')+ throw new GfsError('upload_aborted', 'upload session was canceled')+ if (locked.state === 'expired')+ throw new GfsError('upload_expired', 'upload session has expired')+ throw new GfsError('conflict', `cannot complete upload in ${locked.state}`)+ }+ if (locked.wholeSha256 && expectedWhole && locked.wholeSha256 !== expectedWhole) {+ throw new GfsError('conflict', 'whole-file checksum differs from the session checksum')+ }+ const parts = await client.query(+ `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' ORDER BY part_number`,+ [uploadId]+ )+ const committedParts = parts.rows.map(rowToPart)+ if (committedParts.length !== locked.partCount)+ throw new GfsError('upload_incomplete', 'not all upload parts are committed')+ for (let index = 0; index < locked.partCount; index += 1) {+ const actual = committedParts[index]+ const expected = partGeometry(+ { expectedBytes: locked.expectedBytes, partBytes: locked.partBytes },+ index+ )+ if (+ !actual ||+ actual.partNumber !== expected.partNumber ||+ actual.offsetBytes !== expected.offsetBytes ||+ actual.lengthBytes !== expected.lengthBytes+ ) {+ throw new GfsError(+ 'upload_incomplete',+ `upload part ${index} does not match the session geometry`+ )+ }+ }+ if (+ locked.activePartCount !== 0 ||+ locked.committedPartCount !== locked.partCount ||+ locked.committedBytes !== locked.expectedBytes+ )+ throw new GfsError('upload_incomplete', 'not all upload parts are committed')+ await client.query(+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:finalizers')::bigint)`,+ []+ )+ const finalizing = await client.query(+ `SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'`,+ []+ )+ if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {+ throw new GfsError('quota_exceeded', 'upload finalization concurrency limit reached')+ }+ if (!locked.wholeSha256 && expectedWhole) {+ await client.query(+ `UPDATE gfs_upload_sessions SET whole_sha256 = $2, updated_at = now() WHERE upload_id = $1`,+ [uploadId, expectedWhole]+ )+ }+ await client.query(+ `UPDATE gfs_upload_sessions SET state = 'finalizing', updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,+ [uploadId, locked.sessionEpoch]+ )+ return {+ ...locked,+ wholeSha256: expectedWhole ?? locked.wholeSha256,+ state: 'finalizing' as const,+ parts: committedParts,}}- if (locked.activePartCount !== 0 || locked.committedPartCount !== locked.partCount || locked.committedBytes !== locked.expectedBytes) throw new GfsError('upload_incomplete', 'not all upload parts are committed')- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:finalizers')::bigint)`, [])- const finalizing = await client.query(- `SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'`,- []- )- if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {- throw new GfsError('quota_exceeded', 'upload finalization concurrency limit reached')- }- if (!locked.wholeSha256 && expectedWhole) {- await client.query(`UPDATE gfs_upload_sessions SET whole_sha256 = $2, updated_at = now() WHERE upload_id = $1`, [uploadId, expectedWhole])- }- await client.query(`UPDATE gfs_upload_sessions SET state = 'finalizing', updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`, [uploadId, locked.sessionEpoch])- return { ...locked, wholeSha256: expectedWhole ?? locked.wholeSha256, state: 'finalizing' as const, parts: committedParts }- })+ )if (started.state === 'completed') return toReceipt(started)if (!started.parts) throw new GfsError('internal', 'finalization parts were not loaded')let receipt: { resourceId: string; version: number; sha256: string }const finalizationAbort = new AbortController()const finalizationDeadlineAtMs = this.now() + this.deps.config.finalizeTimeoutMsconst finalizationTimer = setTimeout(- () => finalizationAbort.abort(new GfsError('precondition_failed', 'upload finalization timed out')),+ () =>+ finalizationAbort.abort(+ new GfsError('precondition_failed', 'upload finalization timed out')+ ),Math.max(1, this.deps.config.finalizeTimeoutMs))finalizationTimer.unref?.()@@ -1164,9 +1480,17 @@ export class GfsUploadSessionService {clearTimeout(finalizationTimer)throw new GfsError('upload_finalizing', 'upload finalization is already in progress')}- this.inFlightFinalizers.set(uploadId, { controller: finalizationAbort, sessionEpoch: started.sessionEpoch })+ this.inFlightFinalizers.set(uploadId, {+ controller: finalizationAbort,+ sessionEpoch: started.sessionEpoch,+ })try {- receipt = await this.deps.finalize(started, started.parts, finalizationAbort.signal, finalizationDeadlineAtMs)+ receipt = await this.deps.finalize(+ started,+ started.parts,+ finalizationAbort.signal,+ finalizationDeadlineAtMs+ )} catch (error) {const failureCode = error instanceof GfsError ? error.code : 'finalization_failed'await this.deps.tx.transaction(async client => {@@ -1187,11 +1511,19 @@ export class GfsUploadSessionService {const current = this.inFlightFinalizers.get(uploadId)if (current?.controller === finalizationAbort) this.inFlightFinalizers.delete(uploadId)await this.waitForQuiescence(uploadId)- if (!this.hasInFlight(uploadId) && await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {- await this.deps.tx.transaction(async client => {- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`, [uploadId])- }).catch(() => undefined)+ if (+ !this.hasInFlight(uploadId) &&+ (await removeUploadDirectory(this.deps.storageMountPath, uploadId))+ ) {+ await this.deps.tx+ .transaction(async client => {+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`,+ [uploadId]+ )+ })+ .catch(() => undefined)}throw error} finally {@@ -1204,25 +1536,35 @@ export class GfsUploadSessionService {completed = await this.deps.tx.transaction(async client => {const locked = await this.lockSession(client, uploadId, principal)if (locked.state === 'completed') return locked- if (locked.state !== 'finalizing' || locked.sessionEpoch !== started.sessionEpoch) throw new GfsError('upload_aborted', 'upload session changed during finalization')+ if (locked.state !== 'finalizing' || locked.sessionEpoch !== started.sessionEpoch)+ throw new GfsError('upload_aborted', 'upload session changed during finalization')await client.query(`UPDATE gfs_upload_sessions SET state = 'completed', result_resource_id = $2, result_version = $3, result_sha256 = $4, completed_at = now(), updated_at = now() WHERE upload_id = $1`,[uploadId, receipt.resourceId, receipt.version, receipt.sha256])- return { ...locked, state: 'completed', resultResourceId: receipt.resourceId, resultVersion: receipt.version, resultSha256: receipt.sha256 }+ return {+ ...locked,+ state: 'completed',+ resultResourceId: receipt.resourceId,+ resultVersion: receipt.version,+ resultSha256: receipt.sha256,+ }})} catch (error) {- if (error instanceof CommitOutcomeUnknownError || error instanceof RollbackOutcomeUnknownError) {+ if (+ error instanceof CommitOutcomeUnknownError ||+ error instanceof RollbackOutcomeUnknownError+ ) {const observed = await this.deps.db.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,[uploadId, principal.drive, principal.ownerSubject])const row = observed.rows[0] ? rowToSession(observed.rows[0]) : nullif (- row?.state === 'completed'- && row.resultResourceId === receipt.resourceId- && row.resultVersion === receipt.version- && row.resultSha256 === receipt.sha256+ row?.state === 'completed' &&+ row.resultResourceId === receipt.resourceId &&+ row.resultVersion === receipt.version &&+ row.resultSha256 === receipt.sha256) {completed = row} else {@@ -1233,24 +1575,35 @@ export class GfsUploadSessionService {}}if (await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {- await this.deps.tx.transaction(async client => {- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'completed'`, [uploadId])- }).catch(() => undefined)+ await this.deps.tx+ .transaction(async client => {+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])+ await client.query(+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'completed'`,+ [uploadId]+ )+ })+ .catch(() => undefined)}return toReceipt(completed)}private async loadOwned(uploadId: string, principal: UploadPrincipal): Promise<UploadSessionRow> {requireUuid(uploadId, 'uploadId')- const result = await this.deps.db.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`, [uploadId, principal.drive, principal.ownerSubject])+ const result = await this.deps.db.query(+ `SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,+ [uploadId, principal.drive, principal.ownerSubject]+ )if (!result.rows[0]) throw new GfsError('not_found', 'upload session not found')const session = rowToSession(result.rows[0])assertNotExpired(session, this.now())return session}- private async waitForActivePartsZero(uploadId: string, principal: UploadPrincipal): Promise<void> {+ private async waitForActivePartsZero(+ uploadId: string,+ principal: UploadPrincipal+ ): Promise<void> {const deadline = this.now() + this.deps.config.partTimeoutMsfor (;;) {const result = await this.deps.db.query(@@ -1261,8 +1614,14 @@ export class GfsUploadSessionService {if (!row) throw new GfsError('not_found', 'upload session not found')const localInFlight = this.hasInFlight(uploadId)if (row.activePartCount === 0 && !localInFlight) return- if (this.now() >= deadline) throw new GfsError('precondition_failed', 'upload cancellation timed out while draining in-flight parts')- await new Promise<void>(resolve => setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now()))))+ if (this.now() >= deadline)+ throw new GfsError(+ 'precondition_failed',+ 'upload cancellation timed out while draining in-flight parts'+ )+ await new Promise<void>(resolve =>+ setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now())))+ )}}@@ -1289,13 +1648,14 @@ export class GfsUploadSessionService {if (!result.rows[0]) return nullconst committedPart = rowToPart(result.rows[0])if (- committedPart.state !== 'committed'- || committedPart.sha256 !== checksum- || committedPart.offsetBytes !== part.offsetBytes- || committedPart.lengthBytes !== part.lengthBytes- || committedPart.leaseEpoch !== leaseEpoch- || committedPart.stagingPath !== finalPath- ) return null+ committedPart.state !== 'committed' ||+ committedPart.sha256 !== checksum ||+ committedPart.offsetBytes !== part.offsetBytes ||+ committedPart.lengthBytes !== part.lengthBytes ||+ committedPart.leaseEpoch !== leaseEpoch ||+ committedPart.stagingPath !== finalPath+ )+ return nullconst sessionResult = await this.deps.db.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,[uploadId, principal.drive, principal.ownerSubject]@@ -1306,8 +1666,15 @@ export class GfsUploadSessionService {return { part: committedPart, session }}- private async lockSession(client: TxClient, uploadId: string, principal: UploadPrincipal): Promise<UploadSessionRow> {- const result = await client.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3 FOR UPDATE`, [uploadId, principal.drive, principal.ownerSubject])+ private async lockSession(+ client: TxClient,+ uploadId: string,+ principal: UploadPrincipal+ ): Promise<UploadSessionRow> {+ const result = await client.query(+ `SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3 FOR UPDATE`,+ [uploadId, principal.drive, principal.ownerSubject]+ )if (!result.rows[0]) throw new GfsError('not_found', 'upload session not found')return rowToSession(result.rows[0])}@@ -1318,23 +1685,32 @@ export const GFS_UPLOAD_REQUEST_BODY_LIMITS = {completeBytes: GFS_UPLOAD_V2_COMPLETE_BODY_MAX_BYTES,}-export function uploadCapabilities(enabled: boolean, config: GfsUploadConfig): {+export function uploadCapabilities(+ enabled: boolean,+ config: GfsUploadConfig+): {legacyBase64: { enabled: true; maxFileBytes: number }- resumableV2: ReturnType<typeof disabledGfsUploadV2Capability> | {- enabled: true- maxFileBytes: number- preferredChunkBytes: number- maxChunkBytes: number- maxConcurrentPartsPerSession: number- fallbackConcurrency: number- maxConcurrentPartStreamsGlobal: number- instabilityFailureThreshold: number- sessionTtlSeconds: number- checksumAlgorithms: ['sha256']- wholeFileChecksumRequired: boolean- }+ resumableV2:+ | ReturnType<typeof disabledGfsUploadV2Capability>+ | {+ enabled: true+ maxFileBytes: number+ preferredChunkBytes: number+ maxChunkBytes: number+ maxConcurrentPartsPerSession: number+ fallbackConcurrency: number+ maxConcurrentPartStreamsGlobal: number+ instabilityFailureThreshold: number+ sessionTtlSeconds: number+ checksumAlgorithms: ['sha256']+ wholeFileChecksumRequired: boolean+ }} {- if (!enabled) return { legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 }, resumableV2: disabledGfsUploadV2Capability() }+ if (!enabled)+ return {+ legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 },+ resumableV2: disabledGfsUploadV2Capability(),+ }return {legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 },resumableV2: {diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.shindex 7a3525759..d7c49b88c 100755--- a/scripts/tests/test-gfs-deploy-contract.sh+++ b/scripts/tests/test-gfs-deploy-contract.sh@@ -2,6 +2,19 @@set -euo pipefailROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"; cd "$ROOT"fail() { echo "FAIL: $*" >&2; exit 1; }+external_rest_manifest="$(cat deploy/base/profiles/external-rest-api.yaml)"+profiles_network_policy="$(cat deploy/base/profiles/networkpolicies.yaml)"+ingress_network_policy="$(cat deploy/base/ingress/networkpolicies.yaml)"+[[ "$external_rest_manifest" == *'type: ClusterIP'* ]] \+ || fail 'external-rest-api must remain an internal ClusterIP, not a directly exposed service'+[[ "$profiles_network_policy" == *'name: allow-ingress-profiles'* && \+ "$profiles_network_policy" == *'app: cloudflared'* && \+ "$profiles_network_policy" == *'values: [external-rest-api, profile-ui]'* && \+ "$profiles_network_policy" == *'name: external-rest-api-from-profile-ui'* ]] \+ || fail 'profiles ingress policy does not document the trusted Cloudflare/profile-ui paths'+[[ "$ingress_network_policy" == *'app: external-rest-api'* && \+ "$ingress_network_policy" == *'kubernetes.io/metadata.name: profiles'* ]] \+ || fail 'ingress namespace policy does not restrict cloudflared egress to profiles services'# Scope: OSS-resident GFS deploy invariants only. The GCP deploy invariants# (deploy-dev/deploy-prod workflow ordering, gcp-* Makefile targets, gcp-prod# overlay render, and the HCC rollback contract) live in evenfire-infra after