Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): close upload retry and runtime evidence gaps

Alfredo Lopez3da54c9b34b9
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 596231727..d3423a415 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -182,7 +182,20 @@ jobs:
# suites: the DB migration plus the 7 GFS suites (reader role, blob
# migration, blob reconciler, audit evidence, copy, readiness, structural
# integrity). The matrix `npm test` skips them for lack of the admin URL.
- run: npm test -- --run realPostgres
+ run: |
+ set -euo pipefail
+ log_file="$(mktemp)"
+ trap 'rm -f "$log_file"' EXIT
+ test -n "${CONTROL_API_REAL_PG_ADMIN_URL}"
+ npm test -- --run realPostgres 2>&1 | tee "$log_file"
+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$log_file" || {
+ echo 'real-Postgres control-api lane executed zero passing test files' >&2
+ exit 1
+ }
+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$log_file" || {
+ echo 'real-Postgres control-api lane executed zero passing tests' >&2
+ exit 1
+ }
- name: Install gfs-controller dependencies
working-directory: gfs-controller
@@ -194,7 +207,20 @@ jobs:
CONTROL_API_REAL_PG_ADMIN_URL: postgresql://[email protected]:15433/postgres
# rename-publication concurrency suite — also admin-URL-gated, so the
# gfs-controller matrix entry skips it.
- run: npm test -- --run realPostgres
+ run: |
+ set -euo pipefail
+ log_file="$(mktemp)"
+ trap 'rm -f "$log_file"' EXIT
+ test -n "${CONTROL_API_REAL_PG_ADMIN_URL}"
+ npm test -- --run realPostgres 2>&1 | tee "$log_file"
+ grep -Eq 'Test Files[[:space:]]+[1-9][0-9]* passed' "$log_file" || {
+ echo 'real-Postgres gfs-controller lane executed zero passing test files' >&2
+ exit 1
+ }
+ grep -Eq 'Tests[[:space:]]+[1-9][0-9]* passed' "$log_file" || {
+ echo 'real-Postgres gfs-controller lane executed zero passing tests' >&2
+ exit 1
+ }
# ─── control-ui build + test check ───────────────────────────────────────
# control-ui stays out of the SHARED build/test matrix because its tsconfig
diff --git a/control-ui/lib/__tests__/gfsFileUpload.test.ts b/control-ui/lib/__tests__/gfsFileUpload.test.ts
index 0940eb95a..11ce87cea 100644
--- a/control-ui/lib/__tests__/gfsFileUpload.test.ts
+++ b/control-ui/lib/__tests__/gfsFileUpload.test.ts
@@ -4,6 +4,9 @@ import {
GfsUploadJob,
type GfsUploadStatus,
assertGfsFileUploadSize,
+ isAmbiguousUploadStatus,
+ isRetryableUploadStatus,
+ normalizeInstabilityFailureThreshold,
parseRetryAfter,
} from '@lib/gfsFileUpload'
@@ -27,6 +30,35 @@ describe('parseRetryAfter', () => {
})
})
+describe('normalizeInstabilityFailureThreshold', () => {
+ it('uses the protocol default when older writers omit the field', () => {
+ expect(normalizeInstabilityFailureThreshold(undefined)).toBe(3)
+ })
+
+ it('accepts bounded writer-configured thresholds and rejects invalid values', () => {
+ expect(normalizeInstabilityFailureThreshold(1)).toBe(1)
+ expect(normalizeInstabilityFailureThreshold(5)).toBe(5)
+ expect(() => normalizeInstabilityFailureThreshold(0)).toThrow(/invalid instability threshold/)
+ expect(() => normalizeInstabilityFailureThreshold(101)).toThrow(/invalid instability threshold/)
+ expect(() => normalizeInstabilityFailureThreshold(1.5)).toThrow(/invalid instability threshold/)
+ })
+})
+
+describe('isRetryableUploadStatus', () => {
+ it('uses the same transient allowlist as Desktop and does not retry storage exhaustion', () => {
+ expect(isRetryableUploadStatus(503)).toBe(true)
+ expect(isRetryableUploadStatus(507)).toBe(false)
+ expect(isRetryableUploadStatus(501)).toBe(false)
+ })
+
+ it('only reconciles statuses whose response may follow a committed write', () => {
+ expect(isAmbiguousUploadStatus(500)).toBe(true)
+ expect(isAmbiguousUploadStatus(429)).toBe(false)
+ expect(isAmbiguousUploadStatus(507)).toBe(false)
+ expect(isAmbiguousUploadStatus(501)).toBe(false)
+ })
+})
+
class FakeXhr {
static requests: FakeXhr[] = []
static statuses: number[] = []
diff --git a/control-ui/lib/gfsFileUpload.ts b/control-ui/lib/gfsFileUpload.ts
index a841cbe0c..9fd740f6e 100644
--- a/control-ui/lib/gfsFileUpload.ts
+++ b/control-ui/lib/gfsFileUpload.ts
@@ -9,7 +9,10 @@ import { apiSend } from './api'
const API_BASE = process.env.NEXT_PUBLIC_CONTROL_API_BASE_URL || '/control-api'
const GFS_UPLOAD_DRIVE = 'main'
-const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
+export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
+const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
+export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3
+const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100
export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
@@ -17,6 +20,14 @@ const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 300_000
const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60_000
const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
+export function isRetryableUploadStatus(status: number): boolean {
+ return GFS_UPLOAD_RETRYABLE_STATUS.has(status)
+}
+
+export function isAmbiguousUploadStatus(status: number): boolean {
+ return GFS_UPLOAD_AMBIGUOUS_STATUS.has(status)
+}
+
export interface GfsUploadTarget {
operation: 'create' | 'replace'
parentRid?: string
@@ -129,6 +140,20 @@ export class GfsUploadCapabilityError extends Error {
}
}
+export function normalizeInstabilityFailureThreshold(value: unknown): number {
+ if (value === undefined) return GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD
+ if (
+ !Number.isSafeInteger(value) ||
+ (value as number) < 1 ||
+ (value as number) > GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD
+ ) {
+ throw new GfsUploadCapabilityError(
+ 'Resumable GFS upload capabilities advertised an invalid instability threshold.'
+ )
+ }
+ return value as number
+}
+
export const GFS_LEGACY_UPLOAD_MAX_BYTES = 16 * 1024 * 1024
function uuid(): string {
@@ -263,7 +288,7 @@ function assertRetryable(error: unknown): boolean {
return false
const status =
typeof error === 'object' && error !== null ? (error as { status?: unknown }).status : undefined
- return typeof status === 'number' ? RETRYABLE_STATUS.has(status) : true
+ return typeof status === 'number' ? isRetryableUploadStatus(status) : true
}
function delay(ms: number, signal?: AbortSignal): Promise<void> {
@@ -306,7 +331,7 @@ function ambiguousLifecycleError(error: unknown): boolean {
// Transport failures have no HTTP status; keep them ambiguous so completion
// reconciles durable state instead of assuming that the request was lost.
if (typeof status !== 'number') return true
- return status === 408 || status >= 500
+ return isAmbiguousUploadStatus(status)
}
function terminalReconciliationError(error: unknown): boolean {
@@ -489,7 +514,7 @@ async function loadUploadStatus(
}
function ambiguousPartResponse(status: number | undefined): boolean {
- return status === undefined || status === 408 || status >= 500
+ return status === undefined || isAmbiguousUploadStatus(status)
}
function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {
@@ -683,7 +708,8 @@ async function runParts(
onProgress?: (progress: GfsUploadProgress) => void,
onByteProgress?: (progress: GfsUploadProgress) => void,
onPartFailure?: (partNumber: number) => void,
- fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY
+ fallbackConcurrency = GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY,
+ instabilityFailureThreshold = GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD
): Promise<{ failed: typeof parts; retryableFailures: number; paused: boolean }> {
const initialConcurrency = Math.max(1, Math.min(concurrency, parts.length || 1))
const effectiveFallback = Math.max(
@@ -703,7 +729,11 @@ async function runParts(
const registerRetryableFailure = (): void => {
retryableFailures += 1
consecutiveRetryableFailures += 1
- if (!downgraded && consecutiveRetryableFailures >= 3 && initialConcurrency > effectiveFallback)
+ if (
+ !downgraded &&
+ consecutiveRetryableFailures >= instabilityFailureThreshold &&
+ initialConcurrency > effectiveFallback
+ )
downgraded = true
}
const registerPartSuccess = (): void => {
@@ -1225,6 +1255,9 @@ export class GfsUploadJob {
: GFS_FILE_UPLOAD_DEFAULT_CONCURRENCY
const fallbackConcurrency =
Number(prepared.resumable.fallbackConcurrency) || GFS_FILE_UPLOAD_FALLBACK_CONCURRENCY
+ const instabilityFailureThreshold = normalizeInstabilityFailureThreshold(
+ prepared.resumable.instabilityFailureThreshold
+ )
const result = await runParts(
prepared.session,
this.input.file,
@@ -1238,7 +1271,8 @@ export class GfsUploadJob {
inFlightLoaded.delete(partNumber)
this.uploadedBytes = calculateUploadedBytes()
},
- fallbackConcurrency
+ fallbackConcurrency,
+ instabilityFailureThreshold
)
if (result.paused || this.pauseRequested) {
this.state = 'paused'
diff --git a/desktop-app/src/gfs/upload.test.ts b/desktop-app/src/gfs/upload.test.ts
index 8bf873a08..341ba7d5c 100644
--- a/desktop-app/src/gfs/upload.test.ts
+++ b/desktop-app/src/gfs/upload.test.ts
@@ -3,7 +3,13 @@ import { createHash } from 'node:crypto'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
-import { DesktopGfsUploadJob, uploadLocalFile } from './upload.js'
+import {
+ DesktopGfsUploadJob,
+ isAmbiguousUploadStatus,
+ isRetryableUploadStatus,
+ normalizeInstabilityFailureThreshold,
+ uploadLocalFile,
+} from './upload.js'
function requestPath(url: string): string {
return new URL(url).pathname
@@ -18,6 +24,23 @@ function enabledCapabilities(overrides: Record<string, unknown> = {}) {
}
describe('desktop GFS indexed uploader', () => {
+ it('uses the writer threshold and rejects invalid capability values', () => {
+ expect(normalizeInstabilityFailureThreshold(undefined)).toBe(3)
+ expect(normalizeInstabilityFailureThreshold(1)).toBe(1)
+ expect(normalizeInstabilityFailureThreshold(5)).toBe(5)
+ expect(() => normalizeInstabilityFailureThreshold(0)).toThrow(/invalid instability threshold/)
+ expect(() => normalizeInstabilityFailureThreshold(101)).toThrow(/invalid instability threshold/)
+ })
+
+ it('uses the same transient status allowlist as Control UI', () => {
+ expect(isRetryableUploadStatus(503)).toBe(true)
+ expect(isRetryableUploadStatus(507)).toBe(false)
+ expect(isRetryableUploadStatus(501)).toBe(false)
+ expect(isAmbiguousUploadStatus(500)).toBe(true)
+ expect(isAmbiguousUploadStatus(429)).toBe(false)
+ expect(isAmbiguousUploadStatus(507)).toBe(false)
+ })
+
it('streams independent binary parts with base64 checksums and completes the session', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-upload-'))
try {
diff --git a/desktop-app/src/gfs/upload.ts b/desktop-app/src/gfs/upload.ts
index 9bc2ed225..1a579e165 100644
--- a/desktop-app/src/gfs/upload.ts
+++ b/desktop-app/src/gfs/upload.ts
@@ -11,6 +11,8 @@ const GFS_UPLOAD_V2_FALLBACK_CONCURRENCY = 2
const GFS_UPLOAD_V2_PART_TIMEOUT_MS = 5 * 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_TIMEOUT_MS = 60 * 1000
const GFS_UPLOAD_V2_RECONCILE_ATTEMPTS = 3
+export const GFS_UPLOAD_RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504])
+const GFS_UPLOAD_AMBIGUOUS_STATUS = new Set([408, 500, 502, 503, 504])
export const GFS_UPLOAD_RETRY_MAX_ATTEMPTS = 3
export const GFS_UPLOAD_RETRY_AFTER_CAP_MS = 5_000
const GFS_UPLOAD_RETRY_BASE_DELAY_MS = 250
@@ -85,6 +87,7 @@ interface UploadCapabilities {
preferredChunkBytes?: number
maxChunkBytes?: number
maxConcurrentPartsPerSession?: number
+ instabilityFailureThreshold?: number
fallbackConcurrency?: number
}
}
@@ -115,6 +118,23 @@ export class DesktopUploadCapabilityError extends Error {
}
}
+export const GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD = 3
+const GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD = 100
+
+export function normalizeInstabilityFailureThreshold(value: unknown): number {
+ if (value === undefined) return GFS_UPLOAD_DEFAULT_INSTABILITY_FAILURE_THRESHOLD
+ if (
+ !Number.isSafeInteger(value) ||
+ (value as number) < 1 ||
+ (value as number) > GFS_UPLOAD_MAX_INSTABILITY_FAILURE_THRESHOLD
+ ) {
+ throw new DesktopUploadCapabilityError(
+ 'Resumable GFS upload capabilities advertised an invalid instability threshold.'
+ )
+ }
+ return value as number
+}
+
export interface DesktopUploadTransport {
requestJson<T>(
method: 'GET' | 'POST' | 'HEAD' | 'DELETE',
@@ -169,6 +189,7 @@ export interface DesktopUploadInput {
/** Main-process-only source fence; never crosses the renderer boundary. */
sourceIdentity?: DesktopUploadSourceIdentity
advertisedConcurrency?: number
+ instabilityFailureThreshold?: number
fallbackConcurrency?: number
}
@@ -212,8 +233,16 @@ function unwrap<T>(value: unknown): T {
return envelope.data
}
+export function isRetryableUploadStatus(status: number): boolean {
+ return GFS_UPLOAD_RETRYABLE_STATUS.has(status)
+}
+
+export function isAmbiguousUploadStatus(status: number): boolean {
+ return GFS_UPLOAD_AMBIGUOUS_STATUS.has(status)
+}
+
function retryable(status: number): boolean {
- return status === 408 || status === 425 || status === 429 || status >= 500
+ return isRetryableUploadStatus(status)
}
function retryableError(error: unknown): boolean {
@@ -296,7 +325,7 @@ function ambiguousLifecycleError(error: unknown): boolean {
// Transport failures have no HTTP status; keep them ambiguous so completion
// reconciles durable state instead of assuming that the request was lost.
if (typeof status !== 'number') return true
- return status === 408 || status >= 500
+ return isAmbiguousUploadStatus(status)
}
function terminalReconciliationError(error: unknown): boolean {
@@ -569,7 +598,7 @@ function ambiguousPartResponse(status: number | undefined): boolean {
// A missing response, edge timeout, or proxy/server 5xx may occur after GFSC
// committed the part. 425/429 are explicit admission outcomes and can retry
// without an extra status round-trip.
- return status === undefined || status === 408 || status >= 500
+ return status === undefined || isAmbiguousUploadStatus(status)
}
function partOutcomeUnknown(partNumber: number, cause?: unknown): Error {
@@ -800,7 +829,12 @@ async function runParts(
const registerRetryableFailure = (): void => {
retryableFailures += 1
consecutiveRetryableFailures += 1
- if (!downgraded && consecutiveRetryableFailures >= 3 && concurrency > fallbackConcurrency)
+ if (
+ !downgraded &&
+ consecutiveRetryableFailures >=
+ normalizeInstabilityFailureThreshold(input.instabilityFailureThreshold) &&
+ concurrency > fallbackConcurrency
+ )
downgraded = true
}
const registerPartSuccess = (): void => {
@@ -1286,6 +1320,9 @@ export class DesktopGfsUploadJob {
throw new Error(`GFS files are limited to ${resumable.maxFileBytes} bytes by the writer`)
}
this.input.advertisedConcurrency = resumable.maxConcurrentPartsPerSession
+ this.input.instabilityFailureThreshold = normalizeInstabilityFailureThreshold(
+ resumable.instabilityFailureThreshold
+ )
this.input.fallbackConcurrency = resumable.fallbackConcurrency
const resumeId = this.session?.uploadId ?? this.input.resumeUploadId
if (resumeId) {
diff --git a/docs/architecture/gfs-upload-v2.md b/docs/architecture/gfs-upload-v2.md
index 624096fe2..7448e4f1f 100644
--- a/docs/architecture/gfs-upload-v2.md
+++ b/docs/architecture/gfs-upload-v2.md
@@ -14,23 +14,32 @@ finalization.
This is deliberately not a one-request 200 MiB upload and is not a claim of
tus 1.0 compatibility. It does not raise the legacy JSON/base64 body limit.
-| Contract | Frozen value |
-| --------------------------- | --------------------------------------------------------: |
-| Product maximum | `209715200` bytes (200 MiB binary) |
-| Oversize boundary | `209715201` is rejected before session/payload allocation |
-| Preferred part | `8388608` bytes (8 MiB) |
-| Hard part/request maximum | `16777216` bytes (16 MiB) |
-| Default session concurrency | 4 parts |
-| Instability fallback | 2 parts after 3 consecutive retryable failures/timeouts |
-| Legacy raw-file limit | `16777216` bytes (16 MiB), unchanged during rollout |
-| Legacy GFSC body cap | `25165824` bytes (24 MiB), unchanged during rollout |
-| Protocol arithmetic ceiling | `1073741824` bytes (1 GiB); not an enabled product value |
+| Contract | Frozen value |
+| --------------------------- | --------------------------------------------------------------------------------------------: |
+| Product maximum | `209715200` bytes (200 MiB binary) |
+| Oversize boundary | `209715201` is rejected before session/payload allocation |
+| Preferred part | `8388608` bytes (8 MiB) |
+| Hard part/request maximum | `16777216` bytes (16 MiB) |
+| Default session concurrency | 4 parts |
+| Instability fallback | 2 parts after the advertised threshold (default 3) of consecutive retryable failures/timeouts |
+| Legacy raw-file limit | `16777216` bytes (16 MiB), unchanged during rollout |
+| Legacy GFSC body cap | `25165824` bytes (24 MiB), unchanged during rollout |
+| Protocol arithmetic ceiling | `1073741824` bytes (1 GiB); not an enabled product value |
Cloudflare evaluates the body of each HTTP request. The aggregate file limit
is therefore independent of the request limit: every v2 data-bearing request
is at most 16 MiB, and the verified zone setting can force a lower negotiated
`maxChunkBytes`. No client may send a single 100 MB or 200 MiB body.
+The capability response owns the instability threshold. It is an integer from
+1 through 100; omitted legacy responses mean `3`. Both clients use the
+advertised value when deciding when to reduce concurrency and reject a malformed
+value rather than guessing. The retryable HTTP status allowlist is deliberately
+finite and identical in Desktop and Control UI: `408`, `425`, `429`, `500`,
+`502`, `503`, and `504`. `507 Insufficient Storage` and every other `5xx` are
+terminal until an explicit server-side recovery/retry is requested. A server
+may return `Retry-After`; clients honor it within their bounded retry budget.
+
## Edge admission limits
Upload-v2 has a transport-abuse admission layer before Control API forwards a
diff --git a/docs/testing/gfs-upload-v2.md b/docs/testing/gfs-upload-v2.md
index cde794197..5aace3009 100644
--- a/docs/testing/gfs-upload-v2.md
+++ b/docs/testing/gfs-upload-v2.md
@@ -35,7 +35,9 @@ T0 must execute real tests and fail loudly if zero tests run. It covers:
part PUT, pause/resume, complete, and DELETE;
- no v2 whole-file `Buffer`/`ArrayBuffer`/base64/JSON/IPC value;
- retry classification, response-loss reconciliation, contiguous-vs-committed
- progress, four-to-two fallback, and Desktop IPC byte isolation;
+ progress, four-to-two fallback using the advertised instability threshold,
+ the identical finite retry-status allowlist (including terminal `507`), and
+ Desktop IPC byte isolation;
- proxy invariants: exact v2 prefix bypasses the broad JSON parser, declared
and observed 16 MiB caps are enforced, `Expect` is stripped, upload headers
survive, and disconnect aborts upstream.
@@ -48,6 +50,9 @@ T0 must execute real tests and fail loudly if zero tests run. It covers:
- Desktop state-v2 migration and legacy quarantine, exact owner/team/env/base/
drive resume matching, A → logout → B isolation, epoch-stale dispatch
rejection, and `suspended_auth` requiring explicit same-scope resume;
+- active-part recovery after an unknown commit must decrement
+ `active_part_count` exactly once, including when the reconciliation read also
+ fails; the durable row state, not the transport outcome, is authoritative;
- legacy fallback symlink/path-swap rejection and proof that logout aborts and
awaits a pending fallback before clearing the captured credential.
@@ -78,6 +83,10 @@ T1 uses real Postgres for migration and role checks. It proves:
- same name/size/mtime with different bytes cannot adopt a committed part or
complete a resumed session.
+The dedicated real-Postgres lane is fail-loud: it must report at least one
+executed test file and one passed test. A successful job with zero executed or
+zero passed tests is a failure, not a skipped/green result.
+
The 512 MiB/1 GiB arithmetic ceiling is tested only under an explicit protocol
test configuration. It must not become an enabled 200 MiB product capability.
diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.ts
index b2d503918..6c7e02a89 100644
--- a/gfs-controller/src/config.test.ts
+++ b/gfs-controller/src/config.test.ts
@@ -12,11 +12,14 @@ describe('GFS_STORAGE_ROLE', () => {
expect(loadConfig().storageRole).toBe(role)
})
- it.each([undefined, ''])('rejects absent or empty role %s instead of granting writer authority', role => {
- vi.stubEnv('GFS_DEV_MODE', 'true')
- vi.stubEnv('GFS_STORAGE_ROLE', role)
- expect(() => loadConfig()).toThrow(/GFS_STORAGE_ROLE must be explicitly set/)
- })
+ it.each([undefined, ''])(
+ 'rejects absent or empty role %s instead of granting writer authority',
+ role => {
+ vi.stubEnv('GFS_DEV_MODE', 'true')
+ vi.stubEnv('GFS_STORAGE_ROLE', role)
+ expect(() => loadConfig()).toThrow(/GFS_STORAGE_ROLE must be explicitly set/)
+ }
+ )
it.each(['Writer', 'read', 'writer ', 'unknown'])('rejects unknown value %s', role => {
vi.stubEnv('GFS_DEV_MODE', 'true')
@@ -73,17 +76,7 @@ describe('GFS_SYNC_COPY_*', () => {
'GFS_SYNC_RENAME_MAX_OBJECTS',
'GFS_SYNC_RENAME_TIMEOUT_MS',
])('rejects invalid explicitly configured values for %s', name => {
- for (const raw of [
- '',
- ' ',
- '0',
- '-1',
- '1.5',
- 'NaN',
- 'Infinity',
- '1e3',
- '9007199254740992',
- ]) {
+ for (const raw of ['', ' ', '0', '-1', '1.5', 'NaN', 'Infinity', '1e3', '9007199254740992']) {
expect(() => configWith(name, raw), `${name}=${JSON.stringify(raw)}`).toThrow(
new RegExp(`${name} must be a positive safe integer`)
)
@@ -165,4 +158,16 @@ describe('GFS_UPLOAD_V2 strict disabled contract', () => {
vi.stubEnv('GFS_UPLOAD_FALLBACK_CONCURRENCY', '4')
expect(() => loadConfig()).toThrow(/FALLBACK_CONCURRENCY must be lower/)
})
+
+ it('accepts the bounded instability threshold and rejects values outside the capability contract', () => {
+ vi.stubEnv('GFS_DEV_MODE', 'true')
+ vi.stubEnv('GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD', '7')
+ expect(loadConfig().uploadV2.instabilityFailureThreshold).toBe(7)
+ for (const raw of ['0', '101', '1.5', 'NaN', 'Infinity']) {
+ vi.stubEnv('GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD', raw)
+ expect(() => loadConfig(), `threshold=${raw}`).toThrow(
+ /GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD must be an integer from 1 through 100/
+ )
+ }
+ })
})
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index d4698f322..bb561f9ee 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -1,29 +1,44 @@
-import { mkdtemp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
+import { afterEach, describe, expect, it } from 'vitest'
import { createHash } from 'node:crypto'
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Readable } from 'node:stream'
-import { afterEach, describe, expect, it } from 'vitest'
-import { GfsUploadSessionService, type UploadSessionServiceDeps } from './uploadSession'
-import { partGeometry } from './protocol'
import type { GfsUploadConfig } from '../config'
+import {
+ CommitOutcomeUnknownError,
+ type TransactionOptions,
+ type Transactor,
+ type TxClient,
+} from '../db/writeStore'
+import { partGeometry } from './protocol'
+import { GfsUploadSessionService, type UploadSessionServiceDeps } from './uploadSession'
type Row = Record<string, unknown>
class MemoryDb {
sessions: Row[] = []
parts: Row[] = []
+ failReconciliationReads = false
async query(text: string, values: unknown[] = []): Promise<{ rows: Row[] }> {
const sql = text.replace(/\s+/g, ' ').trim()
if (sql.startsWith('SELECT pg_advisory_xact_lock')) return { rows: [] }
if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE owner_subject')) {
const [owner, drive, key] = values.map(String)
- return { rows: this.sessions.filter(row => row.owner_subject === owner && row.drive === drive && row.idempotency_key === key) }
+ return {
+ rows: this.sessions.filter(
+ row => row.owner_subject === owner && row.drive === drive && row.idempotency_key === key
+ ),
+ }
}
if (sql.startsWith('SELECT * FROM gfs_upload_sessions WHERE upload_id')) {
const [id, drive, owner] = values.map(String)
- return { rows: this.sessions.filter(row => row.upload_id === id && row.drive === drive && row.owner_subject === owner) }
+ return {
+ rows: this.sessions.filter(
+ row => row.upload_id === id && row.drive === drive && row.owner_subject === owner
+ ),
+ }
}
if (sql.startsWith('SELECT session_epoch, state FROM gfs_upload_sessions WHERE upload_id')) {
const [id, drive, owner] = values.map(String)
@@ -33,87 +48,210 @@ class MemoryDb {
.map(row => ({ session_epoch: row.session_epoch, state: row.state })),
}
}
- if (sql.startsWith('SELECT p.upload_id, p.part_number, p.staging_path FROM gfs_upload_parts p')) {
+ if (
+ sql.startsWith('SELECT p.upload_id, p.part_number, p.staging_path FROM gfs_upload_parts p')
+ ) {
return { rows: [] }
}
- if (sql.startsWith("SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()")) {
+ if (
+ sql.startsWith(
+ "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused','finalizing') AND expires_at <= now()"
+ )
+ ) {
return { rows: [] }
}
if (sql.startsWith('SELECT p.upload_id, p.part_number, p.offset_bytes, p.length_bytes')) {
return { rows: [] }
}
- if (sql.startsWith("SELECT upload_id, state FROM gfs_upload_sessions WHERE state IN ('aborted', 'expired', 'failed', 'completed') AND cleanup_at IS NULL")) {
+ if (
+ sql.startsWith(
+ "SELECT upload_id, state FROM gfs_upload_sessions WHERE state IN ('aborted', 'expired', 'failed', 'completed') AND cleanup_at IS NULL"
+ )
+ ) {
return {
rows: this.sessions
- .filter(row => ['aborted', 'expired', 'failed', 'completed'].includes(String(row.state)) && row.cleanup_at == null)
+ .filter(
+ row =>
+ ['aborted', 'expired', 'failed', 'completed'].includes(String(row.state)) &&
+ row.cleanup_at == null
+ )
.map(row => ({ upload_id: row.upload_id, state: row.state })),
}
}
- if (sql.startsWith("SELECT upload_id FROM gfs_upload_sessions WHERE state = 'completed' AND cleanup_at <= now()")) {
+ if (
+ sql.startsWith(
+ "SELECT upload_id FROM gfs_upload_sessions WHERE state = 'completed' AND cleanup_at <= now()"
+ )
+ ) {
return { rows: [] }
}
- if (sql.startsWith('SELECT COALESCE(SUM((expected_bytes - committed_bytes) + expected_bytes)')) {
+ if (
+ sql.startsWith('SELECT COALESCE(SUM((expected_bytes - committed_bytes) + expected_bytes)')
+ ) {
const drive = String(values[0])
const reserved = this.sessions
- .filter(row => row.drive === drive && ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state)))
- .reduce((sum, row) => sum + (Number(row.expected_bytes) - Number(row.committed_bytes ?? 0)) + Number(row.expected_bytes), 0)
+ .filter(
+ row =>
+ row.drive === drive &&
+ ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state))
+ )
+ .reduce(
+ (sum, row) =>
+ sum +
+ (Number(row.expected_bytes) - Number(row.committed_bytes ?? 0)) +
+ Number(row.expected_bytes),
+ 0
+ )
return { rows: [{ reserved: String(reserved) }] }
}
- if (sql === "SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'") {
- return { rows: [{ count: String(this.sessions.filter(row => row.state === 'finalizing').length) }] }
+ if (
+ sql === "SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'"
+ ) {
+ return {
+ rows: [{ count: String(this.sessions.filter(row => row.state === 'finalizing').length) }],
+ }
}
if (sql.startsWith('SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions')) {
- const [drive, owner] = values.map(value => value === undefined ? undefined : String(value))
- const active = this.sessions.filter(row =>
- ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state))
- && (drive === undefined || row.drive === drive)
- && (owner === undefined || row.owner_subject === owner)
+ const [drive, owner] = values.map(value => (value === undefined ? undefined : String(value)))
+ const active = this.sessions.filter(
+ row =>
+ ['initiated', 'uploading', 'paused', 'finalizing'].includes(String(row.state)) &&
+ (drive === undefined || row.drive === drive) &&
+ (owner === undefined || row.owner_subject === owner)
).length
return { rows: [{ count: String(active) }] }
}
if (sql.startsWith('SELECT COUNT(*)::bigint')) {
const drive = String(values[0])
- return { rows: [{ count: String(this.parts.filter(part => part.state === 'reserved' && this.sessions.some(session => session.upload_id === part.upload_id && session.drive === drive)).length) }] }
+ return {
+ rows: [
+ {
+ count: String(
+ this.parts.filter(
+ part =>
+ part.state === 'reserved' &&
+ this.sessions.some(
+ session => session.upload_id === part.upload_id && session.drive === drive
+ )
+ ).length
+ ),
+ },
+ ],
+ }
}
if (sql.startsWith('INSERT INTO gfs_upload_sessions')) {
- const [uploadId, key, drive, owner, primary, operation, fingerprint, parent, resource, name, ifMatch, expected, partBytes, partCount, sha, expiresAt] = values
+ const [
+ uploadId,
+ key,
+ drive,
+ owner,
+ primary,
+ operation,
+ fingerprint,
+ parent,
+ resource,
+ name,
+ ifMatch,
+ expected,
+ partBytes,
+ partCount,
+ sha,
+ expiresAt,
+ ] = values
const row: Row = {
- upload_id: String(uploadId), idempotency_key: String(key), drive: String(drive), owner_subject: String(owner), primary_subject: String(primary),
- operation: String(operation), request_fingerprint: String(fingerprint), parent_rid: parent, resource_rid: resource,
- resource_name: name, if_match: ifMatch, expected_bytes: expected, part_bytes: partBytes, part_count: partCount,
- whole_sha256: sha, committed_bytes: 0, contiguous_bytes: 0, committed_part_count: 0, active_part_count: 0,
- session_epoch: 0, state: 'initiated', result_resource_id: null, result_version: null, result_sha256: null,
- failure_code: null, expires_at: expiresAt, completed_at: null,
+ upload_id: String(uploadId),
+ idempotency_key: String(key),
+ drive: String(drive),
+ owner_subject: String(owner),
+ primary_subject: String(primary),
+ operation: String(operation),
+ request_fingerprint: String(fingerprint),
+ parent_rid: parent,
+ resource_rid: resource,
+ resource_name: name,
+ if_match: ifMatch,
+ expected_bytes: expected,
+ part_bytes: partBytes,
+ part_count: partCount,
+ whole_sha256: sha,
+ committed_bytes: 0,
+ contiguous_bytes: 0,
+ committed_part_count: 0,
+ active_part_count: 0,
+ session_epoch: 0,
+ state: 'initiated',
+ result_resource_id: null,
+ result_version: null,
+ result_sha256: null,
+ failure_code: null,
+ expires_at: expiresAt,
+ completed_at: null,
cleanup_at: null,
}
this.sessions.push(row)
return { rows: [row] }
}
if (sql.startsWith('SELECT upload_id, part_number')) {
+ if (
+ this.failReconciliationReads &&
+ sql.includes('offset_bytes, length_bytes, sha256, state')
+ ) {
+ throw new Error('synthetic reconciliation read failure')
+ }
const [uploadId, partNumber] = values
- const rows = this.parts.filter(part => part.upload_id === String(uploadId) && (partNumber === undefined || Number(part.part_number) === Number(partNumber)) && (!sql.includes("state = 'committed'") || part.state === 'committed'))
+ const rows = this.parts.filter(
+ part =>
+ part.upload_id === String(uploadId) &&
+ (partNumber === undefined || Number(part.part_number) === Number(partNumber)) &&
+ (!sql.includes("state = 'committed'") || part.state === 'committed')
+ )
return { rows }
}
if (sql.startsWith('INSERT INTO gfs_upload_parts')) {
const [uploadId, partNumber, offset, length, sha, staging, epoch] = values
- const row: Row = { upload_id: String(uploadId), part_number: partNumber, offset_bytes: offset, length_bytes: length, sha256: sha, state: 'reserved', staging_path: staging, lease_epoch: epoch, committed_at: null }
+ const row: Row = {
+ upload_id: String(uploadId),
+ part_number: partNumber,
+ offset_bytes: offset,
+ length_bytes: length,
+ sha256: sha,
+ state: 'reserved',
+ staging_path: staging,
+ lease_epoch: epoch,
+ committed_at: null,
+ }
this.parts.push(row)
return { rows: [] }
}
if (sql.startsWith('UPDATE gfs_upload_parts SET offset_bytes')) {
const [uploadId, partNumber, offset, length, sha, staging, epoch] = values
- const row = this.parts.find(part => part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber))!
- Object.assign(row, { offset_bytes: offset, length_bytes: length, sha256: sha, state: 'reserved', staging_path: staging, lease_epoch: epoch, committed_at: null })
+ const row = this.parts.find(
+ part =>
+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
+ )!
+ Object.assign(row, {
+ offset_bytes: offset,
+ length_bytes: length,
+ sha256: sha,
+ state: 'reserved',
+ staging_path: staging,
+ lease_epoch: epoch,
+ committed_at: null,
+ })
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'uploading'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
- row.state = 'uploading'; row.active_part_count = Number(row.active_part_count) + 1
+ row.state = 'uploading'
+ row.active_part_count = Number(row.active_part_count) + 1
return { rows: [] }
}
- if (sql.startsWith('UPDATE gfs_upload_parts SET state = \'committed\'')) {
+ if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'committed'")) {
const [uploadId, partNumber, path] = values
- const row = this.parts.find(part => part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber))!
+ const row = this.parts.find(
+ part =>
+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
+ )!
Object.assign(row, { state: 'committed', staging_path: path })
return { rows: [] }
}
@@ -125,7 +263,12 @@ class MemoryDb {
row.active_part_count = Math.max(0, Number(row.active_part_count) - 1)
let contiguous = 0
for (let number = 0; number < Number(row.part_count); number += 1) {
- const part = this.parts.find(candidate => candidate.upload_id === row.upload_id && Number(candidate.part_number) === number && candidate.state === 'committed')
+ const part = this.parts.find(
+ candidate =>
+ candidate.upload_id === row.upload_id &&
+ Number(candidate.part_number) === number &&
+ candidate.state === 'committed'
+ )
if (!part) break
contiguous += Number(part.length_bytes)
}
@@ -134,8 +277,15 @@ class MemoryDb {
}
if (sql.startsWith("UPDATE gfs_upload_parts SET state = 'failed'")) {
const uploadId = String(values[0])
- for (const row of this.parts.filter(part => part.upload_id === uploadId && part.state === 'reserved')) row.state = 'failed'
- return { rows: [] }
+ const row = this.parts.find(
+ part =>
+ part.upload_id === uploadId &&
+ part.state === 'reserved' &&
+ Number(part.part_number) === Number(values[1])
+ )
+ if (!row) return { rows: [] }
+ row.state = 'failed'
+ return { rows: [{ updated: 1 }] }
}
if (sql.startsWith('UPDATE gfs_upload_sessions SET active_part_count')) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
@@ -154,7 +304,8 @@ class MemoryDb {
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'aborted'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
- row.state = 'aborted'; row.session_epoch = Number(row.session_epoch) + 1
+ row.state = 'aborted'
+ row.session_epoch = Number(row.session_epoch) + 1
return { rows: [] }
}
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {
@@ -222,11 +373,11 @@ afterEach(async () => {
function service(
db: MemoryDb,
config: GfsUploadConfig = TEST_CONFIG,
- overrides: Partial<Pick<UploadSessionServiceDeps, 'now' | 'finalize'>> = {},
+ overrides: Partial<Pick<UploadSessionServiceDeps, 'now' | 'finalize' | 'tx'>> = {}
): GfsUploadSessionService {
return new GfsUploadSessionService({
db,
- tx: { transaction: async fn => fn(db) },
+ tx: overrides.tx ?? { transaction: async fn => fn(db) },
blobs: { availableBytes: async () => 10n ** 12n } as never,
storageMountPath: tempRoot,
config,
@@ -234,17 +385,42 @@ function service(
})
}
-async function settleWithin<T>(promise: Promise<T>, timeoutMs = 500): Promise<
- | { kind: 'completed'; value: T }
- | { kind: 'rejected'; error: unknown }
- | { kind: 'timeout' }
+class ThrowAfterCommitTransactor implements Transactor {
+ private transactionCount = 0
+
+ constructor(
+ private readonly delegate: Transactor,
+ private readonly afterCommit: () => void
+ ) {}
+
+ async transaction<T>(
+ fn: (client: TxClient) => Promise<T>,
+ options?: TransactionOptions
+ ): Promise<T> {
+ this.transactionCount += 1
+ const result = await this.delegate.transaction(fn, options)
+ // create/reservation is the first transaction; the second is the part
+ // commit. The callback makes the subsequent reconciliation read fail.
+ if (this.transactionCount === 2) {
+ this.afterCommit()
+ throw new CommitOutcomeUnknownError(new Error('synthetic post-commit response loss'))
+ }
+ return result
+ }
+}
+
+async function settleWithin<T>(
+ promise: Promise<T>,
+ timeoutMs = 500
+): Promise<
+ { kind: 'completed'; value: T } | { kind: 'rejected'; error: unknown } | { kind: 'timeout' }
> {
let timer: ReturnType<typeof setTimeout> | undefined
try {
return await Promise.race([
promise.then(
value => ({ kind: 'completed' as const, value }),
- error => ({ kind: 'rejected' as const, error }),
+ error => ({ kind: 'rejected' as const, error })
),
new Promise<{ kind: 'timeout' }>(resolve => {
timer = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs)
@@ -256,40 +432,152 @@ async function settleWithin<T>(promise: Promise<T>, timeoutMs = 500): Promise<
}
describe('GfsUploadSessionService', () => {
- it('creates idempotent sessions and keeps indexed commits separate from contiguous progress', { timeout: 15_000 }, async () => {
+ it(
+ 'creates idempotent sessions and keeps indexed commits separate from contiguous progress',
+ { timeout: 15_000 },
+ async () => {
+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
+ const db = new MemoryDb()
+ const uploads = service(db)
+ const created = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'data.bin',
+ sizeBytes: 2097152,
+ idempotencyKey: key,
+ })
+ expect(created.created).toBe(true)
+ const replay = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'data.bin',
+ sizeBytes: 2097152,
+ idempotencyKey: key,
+ })
+ expect(replay.created).toBe(false)
+ expect(replay.session.uploadId).toBe(created.session.uploadId)
+
+ const first = Buffer.alloc(1048576, 0x11)
+ const second = Buffer.alloc(1048576, 0x22)
+ const firstGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 0)
+ const secondGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 1)
+ const checksum = (value: Buffer) => createSha(value)
+ const outOfOrder = await uploads.putPart(
+ created.session.uploadId,
+ secondGeometry,
+ checksum(second),
+ Readable.from([second]) as never,
+ principal
+ )
+ expect(outOfOrder.session.contiguousBytes).toBe(0)
+ const ordered = await uploads.putPart(
+ created.session.uploadId,
+ firstGeometry,
+ checksum(first),
+ Readable.from([first]) as never,
+ principal
+ )
+ expect(ordered.session.contiguousBytes).toBe(2097152)
+ expect(ordered.session.committedPartCount).toBe(2)
+ expect(await readFile(ordered.part.stagingPath)).toEqual(first)
+ const duplicate = await uploads.putPart(
+ created.session.uploadId,
+ firstGeometry,
+ checksum(first),
+ Readable.from([first]) as never,
+ principal
+ )
+ expect(duplicate.part.sha256).toBe(checksum(first))
+ }
+ )
+
+ it('does not double-decrement active parts after an unknown commit and failed reconciliation', async () => {
tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
const db = new MemoryDb()
- const uploads = service(db)
- const created = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 2097152, idempotencyKey: key })
- expect(created.created).toBe(true)
- const replay = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 2097152, idempotencyKey: key })
- expect(replay.created).toBe(false)
- expect(replay.session.uploadId).toBe(created.session.uploadId)
+ const created = await service(db).create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'counter-safe.bin',
+ sizeBytes: 2 * TEST_CONFIG.preferredPartBytes,
+ idempotencyKey: '66666666-6666-4666-8666-666666666666',
+ })
+ const session = db.sessions[0]!
+ session.active_part_count = 1
+ db.parts.push({
+ upload_id: created.session.uploadId,
+ part_number: 1,
+ offset_bytes: TEST_CONFIG.preferredPartBytes,
+ length_bytes: TEST_CONFIG.preferredPartBytes,
+ sha256: 'f'.repeat(64),
+ state: 'reserved',
+ staging_path: join(tempRoot, 'other.part'),
+ lease_epoch: 0,
+ committed_at: null,
+ })
+
+ const tx = new ThrowAfterCommitTransactor({ transaction: async fn => fn(db) }, () => {
+ db.failReconciliationReads = true
+ })
+ const uploads = service(db, TEST_CONFIG, { tx })
+ const first = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x31)
+ await expect(
+ uploads.putPart(
+ created.session.uploadId,
+ partGeometry(
+ {
+ expectedBytes: 2 * TEST_CONFIG.preferredPartBytes,
+ partBytes: TEST_CONFIG.preferredPartBytes,
+ },
+ 0
+ ),
+ createSha(first),
+ Readable.from([first]) as never,
+ principal
+ )
+ ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)
- const first = Buffer.alloc(1048576, 0x11)
- const second = Buffer.alloc(1048576, 0x22)
- const firstGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 0)
- const secondGeometry = partGeometry({ expectedBytes: 2097152, partBytes: 1048576 }, 1)
- const checksum = (value: Buffer) => createSha(value)
- const outOfOrder = await uploads.putPart(created.session.uploadId, secondGeometry, checksum(second), Readable.from([second]) as never, principal)
- expect(outOfOrder.session.contiguousBytes).toBe(0)
- const ordered = await uploads.putPart(created.session.uploadId, firstGeometry, checksum(first), Readable.from([first]) as never, principal)
- expect(ordered.session.contiguousBytes).toBe(2097152)
- expect(ordered.session.committedPartCount).toBe(2)
- expect(await readFile(ordered.part.stagingPath)).toEqual(first)
- const duplicate = await uploads.putPart(created.session.uploadId, firstGeometry, checksum(first), Readable.from([first]) as never, principal)
- expect(duplicate.part.sha256).toBe(checksum(first))
+ expect(session.active_part_count).toBe(1)
+ expect(db.parts.find(part => Number(part.part_number) === 0)?.state).toBe('committed')
+ expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')
})
it('cancels terminally and rejects a conflicting idempotency fingerprint', async () => {
tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
const db = new MemoryDb()
const uploads = service(db)
- await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'data.bin', sizeBytes: 0, idempotencyKey: key })
- await expect(uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'other.bin', sizeBytes: 0, idempotencyKey: key })).rejects.toMatchObject({ code: 'idempotency_conflict' })
- const session = await uploads.create({ ...principal, operation: 'create', parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', name: 'cancel.bin', sizeBytes: 0, idempotencyKey: '22222222-2222-4222-8222-222222222222' })
+ await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'data.bin',
+ sizeBytes: 0,
+ idempotencyKey: key,
+ })
+ await expect(
+ uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'other.bin',
+ sizeBytes: 0,
+ idempotencyKey: key,
+ })
+ ).rejects.toMatchObject({ code: 'idempotency_conflict' })
+ const session = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'cancel.bin',
+ sizeBytes: 0,
+ idempotencyKey: '22222222-2222-4222-8222-222222222222',
+ })
await uploads.cancel(session.session.uploadId, principal)
- await expect(uploads.get(session.session.uploadId, principal)).rejects.toMatchObject({ code: 'upload_aborted' })
+ await expect(uploads.get(session.session.uploadId, principal)).rejects.toMatchObject({
+ code: 'upload_aborted',
+ })
})
it('releases a rejected finalizer before quiescence and cleans the failed upload immediately', async () => {
@@ -302,7 +590,9 @@ describe('GfsUploadSessionService', () => {
// five-minute part timeout: the old code still skipped cleanup because
// the failed finalizer remained registered while it drained itself.
now: () => (now += TEST_CONFIG.partTimeoutMs + 1),
- finalize: async () => { throw new Error('synthetic finalizer rejection') },
+ finalize: async () => {
+ throw new Error('synthetic finalizer rejection')
+ },
})
const created = await uploads.create({
...principal,
@@ -319,14 +609,17 @@ describe('GfsUploadSessionService', () => {
const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))
expect(outcome.kind).toBe('rejected')
- if (outcome.kind !== 'rejected') throw new Error(`unexpected completion outcome: ${outcome.kind}`)
+ if (outcome.kind !== 'rejected')
+ throw new Error(`unexpected completion outcome: ${outcome.kind}`)
expect(outcome.error).toMatchObject({ message: 'synthetic finalizer rejection' })
expect(db.sessions[0]).toMatchObject({
state: 'failed',
failure_code: 'finalization_failed',
cleanup_at: expect.any(String),
})
- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)
+ expect(
+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
+ ).toBe(0)
expect(db.parts).toHaveLength(0)
await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })
})
@@ -338,9 +631,10 @@ describe('GfsUploadSessionService', () => {
const config = { ...TEST_CONFIG, finalizeTimeoutMs: 5 }
const uploads = service(db, config, {
now: () => (now += config.partTimeoutMs + 1),
- finalize: async (_session, _parts, signal) => new Promise((_resolve, reject) => {
- signal?.addEventListener('abort', () => reject(signal.reason), { once: true })
- }),
+ finalize: async (_session, _parts, signal) =>
+ new Promise((_resolve, reject) => {
+ signal?.addEventListener('abort', () => reject(signal.reason), { once: true })
+ }),
})
const created = await uploads.create({
...principal,
@@ -357,14 +651,17 @@ describe('GfsUploadSessionService', () => {
const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))
expect(outcome.kind).toBe('rejected')
- if (outcome.kind !== 'rejected') throw new Error(`unexpected completion outcome: ${outcome.kind}`)
+ if (outcome.kind !== 'rejected')
+ throw new Error(`unexpected completion outcome: ${outcome.kind}`)
expect(outcome.error).toMatchObject({ code: 'precondition_failed' })
expect(db.sessions[0]).toMatchObject({
state: 'failed',
failure_code: 'precondition_failed',
cleanup_at: expect.any(String),
})
- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)
+ expect(
+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
+ ).toBe(0)
expect(db.parts).toHaveLength(0)
await expect(stat(uploadDirectory)).rejects.toMatchObject({ code: 'ENOENT' })
})
@@ -375,7 +672,9 @@ describe('GfsUploadSessionService', () => {
let now = 0
const uploads = service(db, TEST_CONFIG, {
now: () => (now += TEST_CONFIG.partTimeoutMs + 1),
- finalize: async () => { throw new Error('synthetic cleanup retry') },
+ finalize: async () => {
+ throw new Error('synthetic cleanup retry')
+ },
})
const created = await uploads.create({
...principal,
@@ -394,7 +693,9 @@ describe('GfsUploadSessionService', () => {
const outcome = await settleWithin(uploads.complete(created.session.uploadId, principal))
expect(outcome.kind).toBe('rejected')
expect(db.sessions[0]).toMatchObject({ state: 'failed', cleanup_at: null })
- expect((uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size).toBe(0)
+ expect(
+ (uploads as unknown as { inFlightFinalizers: Map<string, unknown> }).inFlightFinalizers.size
+ ).toBe(0)
await rm(uploadDirectory, { force: true })
await mkdir(uploadDirectory, { recursive: true })
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index 388ccbc1c..e094cfb0a 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -1,26 +1,26 @@
import { createHash, randomUUID } from 'node:crypto'
import { constants } from 'node:fs'
import { lstat, mkdir, open, rename, rm, stat } from 'node:fs/promises'
-import { dirname, resolve } from 'node:path'
import type { IncomingMessage } from 'node:http'
+import { dirname, resolve } from 'node:path'
import { Readable, Transform } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { GfsError } from '../api/errors'
-import { normalizeResourceName, ResourceNameError } from '../api/resourceName'
-import { normalizeResourceId, PathError } from '../storage/paths'
+import { ResourceNameError, normalizeResourceName } from '../api/resourceName'
+import type { GfsUploadConfig } from '../config'
import { CommitOutcomeUnknownError, RollbackOutcomeUnknownError } from '../db/writeStore'
import type { BlobWriter, Transactor, TxClient } from '../db/writeStore'
-import type { GfsUploadConfig } from '../config'
+import { PathError, normalizeResourceId } from '../storage/paths'
import {
GFS_UPLOAD_V2_COMPLETE_BODY_MAX_BYTES,
GFS_UPLOAD_V2_METADATA_BODY_MAX_BYTES,
GFS_UPLOAD_V2_PRODUCT_MAX_BYTES,
+ GfsUploadGeometryError,
+ type GfsUploadPartGeometry,
disabledGfsUploadV2Capability,
partCountFor,
partGeometry,
validatePartGeometry,
- GfsUploadGeometryError,
- type GfsUploadPartGeometry,
} from './protocol'
export type { GfsUploadPartGeometry } from './protocol'
@@ -85,9 +85,18 @@ export interface UploadSessionServiceDeps {
storageMountPath: string
config: GfsUploadConfig
now?: () => number
- authorize?: (principal: UploadPrincipal, operation: 'create' | 'replace', resourceRid?: string) => Promise<void>
+ authorize?: (
+ principal: UploadPrincipal,
+ operation: 'create' | 'replace',
+ resourceRid?: string
+ ) => Promise<void>
/** PR3 supplies this; PR2 intentionally cannot publish a resource. */
- finalize?: (session: UploadSessionRow, parts: UploadPartRow[], signal?: AbortSignal, deadlineAtMs?: number) => Promise<{ resourceId: string; version: number; sha256: string }>
+ finalize?: (
+ session: UploadSessionRow,
+ parts: UploadPartRow[],
+ signal?: AbortSignal,
+ deadlineAtMs?: number
+ ) => Promise<{ resourceId: string; version: number; sha256: string }>
}
export interface UploadSessionRow {
@@ -148,7 +157,8 @@ interface InFlightFinalizer {
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
const SHA256_RE = /^[0-9a-f]{64}$/i
-const TEMP_PART_RE = /^(\d+)\.part\.tmp-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i
+const TEMP_PART_RE =
+ /^(\d+)\.part\.tmp-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i
function rowToSession(row: Record<string, unknown>): UploadSessionRow {
return {
@@ -194,7 +204,11 @@ function rowToPart(row: Record<string, unknown>): UploadPartRow {
}
}
-function canonicalFingerprint(input: CreateUploadSessionInput, partBytes: number, partCount: number): string {
+function canonicalFingerprint(
+ input: CreateUploadSessionInput,
+ partBytes: number,
+ partCount: number
+): string {
return JSON.stringify({
owner: input.ownerSubject,
drive: input.drive,
@@ -236,7 +250,8 @@ function requireUuid(value: string, field: string): string {
}
function requireChecksum(value: string | undefined, field: string): string {
- if (value === undefined || !SHA256_RE.test(value)) throw new GfsError('path_invalid', `${field} must be a SHA-256 hex digest`)
+ if (value === undefined || !SHA256_RE.test(value))
+ throw new GfsError('path_invalid', `${field} must be a SHA-256 hex digest`)
return value.toLowerCase()
}
@@ -265,7 +280,10 @@ function toReceipt(session: UploadSessionRow): UploadSessionReceipt {
}
function assertNotExpired(session: UploadSessionRow, now: number): void {
- if (Date.parse(session.expiresAt) <= now && !['completed', 'aborted', 'expired'].includes(session.state)) {
+ if (
+ Date.parse(session.expiresAt) <= now &&
+ !['completed', 'aborted', 'expired'].includes(session.state)
+ ) {
throw new GfsError('upload_expired', 'upload session has expired')
}
}
@@ -291,7 +309,12 @@ function sessionPath(root: string, uploadId: string, partNumber: number, suffix:
* intentionally rejected because a new lease may own it after the row lock is
* released.
*/
-async function validatedTempPath(root: string, uploadId: string, partNumber: number, candidate: string): Promise<string | null> {
+async function validatedTempPath(
+ root: string,
+ uploadId: string,
+ partNumber: number,
+ candidate: string
+): Promise<string | null> {
const canonicalUploadId = requireUuid(uploadId, 'uploadId')
const expectedDir = resolve(root, '.uploads', canonicalUploadId, 'parts')
const resolved = resolve(candidate)
@@ -299,7 +322,12 @@ async function validatedTempPath(root: string, uploadId: string, partNumber: num
if (!resolved.startsWith(prefix)) return null
const match = TEMP_PART_RE.exec(resolved.slice(prefix.length))
if (!match || Number(match[1]) !== partNumber) return null
- for (const directory of [resolve(root), resolve(root, '.uploads'), resolve(root, '.uploads', canonicalUploadId), expectedDir]) {
+ for (const directory of [
+ resolve(root),
+ resolve(root, '.uploads'),
+ resolve(root, '.uploads', canonicalUploadId),
+ expectedDir,
+ ]) {
try {
if (!(await lstat(directory)).isDirectory()) return null
} catch (error) {
@@ -318,7 +346,8 @@ async function ensureStagingDir(path: string): Promise<void> {
await mkdir(partsDir, { recursive: true, mode: 0o700 })
for (const directory of [root, uploadsDir, uploadDir, partsDir]) {
const info = await lstat(directory)
- if (!info.isDirectory()) throw new GfsError('path_invalid', 'upload staging ancestors must be real directories')
+ if (!info.isDirectory())
+ throw new GfsError('path_invalid', 'upload staging ancestors must be real directories')
}
}
@@ -349,7 +378,12 @@ async function removeUploadDirectory(root: string, uploadId: string): Promise<bo
async function committedPartFilePresent(root: string, part: UploadPartRow): Promise<boolean> {
const expected = sessionPath(root, part.uploadId, part.partNumber, '')
if (part.stagingPath !== expected) return false
- const safe = await validatedTempPath(root, part.uploadId, part.partNumber, `${expected}.tmp-${randomUUID()}`)
+ const safe = await validatedTempPath(
+ root,
+ part.uploadId,
+ part.partNumber,
+ `${expected}.tmp-${randomUUID()}`
+ )
// validatedTempPath also validates all private ancestors. The synthetic
// candidate is never unlinked and only serves as the containment check.
if (!safe) return false
@@ -382,7 +416,11 @@ async function readPartBody(
try {
if (destination) {
await ensureStagingDir(destination)
- handle = await open(destination, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600)
+ handle = await open(
+ destination,
+ constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,
+ 0o600
+ )
stream = handle.createWriteStream()
}
let bytes = 0
@@ -413,11 +451,13 @@ async function readPartBody(
signal?.throwIfAborted()
const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw)
bytes += chunk.length
- if (bytes > expectedLength || bytes > maxLength) throw new GfsError('payload_too_large', 'part body exceeds its declared length')
+ if (bytes > expectedLength || bytes > maxLength)
+ throw new GfsError('payload_too_large', 'part body exceeds its declared length')
digest.update(chunk)
}
}
- if (bytes !== expectedLength) throw new GfsError('path_invalid', `part body has ${bytes} bytes; expected ${expectedLength}`)
+ if (bytes !== expectedLength)
+ throw new GfsError('path_invalid', `part body has ${bytes} bytes; expected ${expectedLength}`)
if (destination) {
const syncHandle = await open(destination, constants.O_RDONLY | constants.O_NOFOLLOW)
try {
@@ -465,28 +505,47 @@ export class GfsUploadSessionService {
}
private hasInFlight(uploadId: string): boolean {
- return [...this.inFlight.keys()].some(key => key.startsWith(`${uploadId}:`))
- || this.inFlightFinalizers.has(uploadId)
+ return (
+ [...this.inFlight.keys()].some(key => key.startsWith(`${uploadId}:`)) ||
+ this.inFlightFinalizers.has(uploadId)
+ )
}
- async create(input: CreateUploadSessionInput): Promise<{ created: boolean; session: UploadSessionReceipt }> {
+ async create(
+ input: CreateUploadSessionInput
+ ): Promise<{ created: boolean; session: UploadSessionReceipt }> {
requireEnabled(this.deps.config)
const idempotencyKey = requireUuid(input.idempotencyKey, 'idempotencyKey')
- if (!Number.isSafeInteger(input.sizeBytes) || input.sizeBytes < 0 || input.sizeBytes > GFS_UPLOAD_V2_PRODUCT_MAX_BYTES) {
- throw new GfsError('payload_too_large', `sizeBytes must be between 0 and ${GFS_UPLOAD_V2_PRODUCT_MAX_BYTES}`)
+ if (
+ !Number.isSafeInteger(input.sizeBytes) ||
+ input.sizeBytes < 0 ||
+ input.sizeBytes > GFS_UPLOAD_V2_PRODUCT_MAX_BYTES
+ ) {
+ throw new GfsError(
+ 'payload_too_large',
+ `sizeBytes must be between 0 and ${GFS_UPLOAD_V2_PRODUCT_MAX_BYTES}`
+ )
}
const name = normalizedName(input.name)
const parentRid = normalizedRid(input.parentRid, 'parentRid')
const resourceRid = normalizedRid(input.resourceRid, 'resourceRid')
- if (input.operation === 'create' && (!parentRid || !name)) throw new GfsError('path_invalid', 'create requires parentRid and name')
- if (input.operation === 'replace' && !resourceRid) throw new GfsError('path_invalid', 'replace requires resourceRid')
- const wholeSha256 = input.wholeSha256 === null || input.wholeSha256 === undefined
- ? null
- : requireChecksum(input.wholeSha256, 'wholeSha256')
- await this.deps.authorize?.(input, input.operation, input.operation === 'create' ? parentRid : resourceRid)
+ if (input.operation === 'create' && (!parentRid || !name))
+ throw new GfsError('path_invalid', 'create requires parentRid and name')
+ if (input.operation === 'replace' && !resourceRid)
+ throw new GfsError('path_invalid', 'replace requires resourceRid')
+ const wholeSha256 =
+ input.wholeSha256 === null || input.wholeSha256 === undefined
+ ? null
+ : requireChecksum(input.wholeSha256, 'wholeSha256')
+ await this.deps.authorize?.(
+ input,
+ input.operation,
+ input.operation === 'create' ? parentRid : resourceRid
+ )
const partBytes = this.deps.config.preferredPartBytes
const partCount = partCountFor({ expectedBytes: input.sizeBytes, partBytes })
- if (partCount > this.deps.config.maxPartCount) throw new GfsError('payload_too_large', 'upload part count exceeds the configured maximum')
+ if (partCount > this.deps.config.maxPartCount)
+ throw new GfsError('payload_too_large', 'upload part count exceeds the configured maximum')
const canonicalInput = { ...input, name, parentRid, resourceRid, wholeSha256 }
const requestFingerprint = canonicalFingerprint(canonicalInput, partBytes, partCount)
const existing = await this.deps.db.query(
@@ -495,7 +554,11 @@ export class GfsUploadSessionService {
)
if (existing.rows[0]) {
const row = rowToSession(existing.rows[0])
- if (row.requestFingerprint !== requestFingerprint) throw new GfsError('idempotency_conflict', 'idempotency key is bound to a different request')
+ if (row.requestFingerprint !== requestFingerprint)
+ throw new GfsError(
+ 'idempotency_conflict',
+ 'idempotency key is bound to a different request'
+ )
return { created: false, session: toReceipt(row) }
}
const uploadId = requireUuid(input.uploadId ?? randomUUID(), 'uploadId')
@@ -503,7 +566,10 @@ export class GfsUploadSessionService {
const available = this.deps.blobs.availableBytes ? await this.deps.blobs.availableBytes() : null
try {
const result = await this.deps.tx.transaction(async client => {
- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:admission:' || $1)::bigint)`, [input.drive])
+ await client.query(
+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:admission:' || $1)::bigint)`,
+ [input.drive]
+ )
const subjectActive = await client.query(
`SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions
WHERE drive = $1 AND owner_subject = $2 AND state IN ('initiated','uploading','paused','finalizing') AND expires_at > now()`,
@@ -528,8 +594,14 @@ export class GfsUploadSessionService {
[input.drive]
)
const reservedBytes = BigInt(String(reserved.rows[0]?.reserved ?? 0))
- if (reservedBytes + BigInt(input.sizeBytes) * 2n + BigInt(this.deps.config.minFreeBytes) > available) {
- throw new GfsError('insufficient_storage', 'insufficient storage for upload reservation')
+ if (
+ reservedBytes + BigInt(input.sizeBytes) * 2n + BigInt(this.deps.config.minFreeBytes) >
+ available
+ ) {
+ throw new GfsError(
+ 'insufficient_storage',
+ 'insufficient storage for upload reservation'
+ )
}
}
const inserted = await client.query(
@@ -540,10 +612,24 @@ export class GfsUploadSessionService {
whole_sha256, state, expires_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,'initiated',$16)
RETURNING *`,
- [uploadId, idempotencyKey, input.drive, input.ownerSubject, input.primarySubject,
- input.operation, requestFingerprint, parentRid ?? null, resourceRid ?? null,
- name ?? null, input.ifMatch ?? null, input.sizeBytes, partBytes, partCount,
- wholeSha256, expiresAt]
+ [
+ uploadId,
+ idempotencyKey,
+ input.drive,
+ input.ownerSubject,
+ input.primarySubject,
+ input.operation,
+ requestFingerprint,
+ parentRid ?? null,
+ resourceRid ?? null,
+ name ?? null,
+ input.ifMatch ?? null,
+ input.sizeBytes,
+ partBytes,
+ partCount,
+ wholeSha256,
+ expiresAt,
+ ]
)
return rowToSession(inserted.rows[0]!)
})
@@ -556,7 +642,11 @@ export class GfsUploadSessionService {
)
const row = replay.rows[0] ? rowToSession(replay.rows[0]) : null
if (!row) throw error
- if (row.requestFingerprint !== requestFingerprint) throw new GfsError('idempotency_conflict', 'idempotency key is bound to a different request')
+ if (row.requestFingerprint !== requestFingerprint)
+ throw new GfsError(
+ 'idempotency_conflict',
+ 'idempotency key is bound to a different request'
+ )
return { created: false, session: toReceipt(row) }
}
}
@@ -584,14 +674,20 @@ export class GfsUploadSessionService {
}
}
- async status(uploadId: string, principal: UploadPrincipal, options?: { cursor?: string; limit?: number }): Promise<UploadStatusPage> {
+ async status(
+ uploadId: string,
+ principal: UploadPrincipal,
+ options?: { cursor?: string; limit?: number }
+ ): Promise<UploadStatusPage> {
requireEnabled(this.deps.config)
const session = await this.loadOwned(uploadId, principal)
const rawCursor = options?.cursor ?? ''
- if (rawCursor && !/^\d+$/.test(rawCursor)) throw new GfsError('path_invalid', 'status cursor is invalid')
+ if (rawCursor && !/^\d+$/.test(rawCursor))
+ throw new GfsError('path_invalid', 'status cursor is invalid')
const cursor = rawCursor ? Number(rawCursor) : -1
const requestedLimit = options?.limit ?? 256
- if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1 || requestedLimit > 256) throw new GfsError('path_invalid', 'status limit is invalid')
+ if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1 || requestedLimit > 256)
+ throw new GfsError('path_invalid', 'status limit is invalid')
const result = await this.deps.db.query(
`SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch
FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' AND part_number > $2 ORDER BY part_number ASC LIMIT $3`,
@@ -625,14 +721,26 @@ export class GfsUploadSessionService {
requireEnabled(this.deps.config)
const expectedChecksum = requireChecksum(checksum, 'Upload-Checksum')
const session = await this.loadOwned(uploadId, principal)
- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')
+ await this.deps.authorize?.(
+ principal,
+ session.operation,
+ session.operation === 'create'
+ ? (session.parentRid ?? undefined)
+ : (session.resourceRid ?? undefined)
+ )
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is pausing')
assertNotExpired(session, this.now())
if (session.state === 'paused') throw new GfsError('conflict', 'upload session is paused')
- if (['finalizing', 'completed'].includes(session.state)) throw new GfsError('upload_completed', 'upload session is no longer writable')
- if (['aborted', 'expired', 'failed'].includes(session.state)) throw new GfsError('upload_aborted', 'upload session is not writable')
+ if (['finalizing', 'completed'].includes(session.state))
+ throw new GfsError('upload_completed', 'upload session is no longer writable')
+ if (['aborted', 'expired', 'failed'].includes(session.state))
+ throw new GfsError('upload_aborted', 'upload session is not writable')
try {
- validatePartGeometry({ expectedBytes: session.expectedBytes, partBytes: session.partBytes }, part)
+ validatePartGeometry(
+ { expectedBytes: session.expectedBytes, partBytes: session.partBytes },
+ part
+ )
} catch (error) {
if (error instanceof GfsUploadGeometryError) throw new GfsError('path_invalid', error.message)
throw error
@@ -653,56 +761,94 @@ export class GfsUploadSessionService {
}
try {
reserved = await this.deps.tx.transaction(async client => {
- const locked = await this.lockSession(client, uploadId, principal)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')
- if (locked.state === 'paused') throw new GfsError('conflict', 'upload session is paused')
- if (locked.state !== 'initiated' && locked.state !== 'uploading') throw new GfsError('upload_aborted', 'upload session is not writable')
- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:streams:' || $1)::bigint)`, [locked.drive])
- const existing = await client.query(
- `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch
+ const locked = await this.lockSession(client, uploadId, principal)
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is pausing')
+ if (locked.state === 'paused') throw new GfsError('conflict', 'upload session is paused')
+ if (locked.state !== 'initiated' && locked.state !== 'uploading')
+ throw new GfsError('upload_aborted', 'upload session is not writable')
+ await client.query(
+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:streams:' || $1)::bigint)`,
+ [locked.drive]
+ )
+ const existing = await client.query(
+ `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch
FROM gfs_upload_parts WHERE upload_id = $1 AND part_number = $2 FOR UPDATE`,
- [uploadId, part.partNumber]
- )
- if (existing.rows[0] && String(existing.rows[0].state) === 'committed') return { session: locked, existing: rowToPart(existing.rows[0]) }
- if (existing.rows[0] && String(existing.rows[0].state) === 'reserved') {
- throw new GfsError('conflict', 'upload part is already in flight')
- }
- const activeGlobal = await client.query(
- `SELECT COUNT(*)::bigint AS count
+ [uploadId, part.partNumber]
+ )
+ if (existing.rows[0] && String(existing.rows[0].state) === 'committed')
+ return { session: locked, existing: rowToPart(existing.rows[0]) }
+ if (existing.rows[0] && String(existing.rows[0].state) === 'reserved') {
+ throw new GfsError('conflict', 'upload part is already in flight')
+ }
+ const activeGlobal = await client.query(
+ `SELECT COUNT(*)::bigint AS count
FROM gfs_upload_parts p
JOIN gfs_upload_sessions s ON s.upload_id = p.upload_id
WHERE s.drive = $1 AND p.state = 'reserved'`,
- [locked.drive]
- )
- if (Number(activeGlobal.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentPartStreamsGlobal) {
- throw new GfsError('quota_exceeded', 'global upload stream concurrency limit reached')
- }
- if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession) throw new GfsError('quota_exceeded', 'per-session part concurrency limit reached')
- const stagingPath = sessionPath(this.deps.storageMountPath, uploadId, part.partNumber, `.tmp-${randomUUID()}`)
- const previousLeaseEpoch = existing.rows[0] ? Number(existing.rows[0].lease_epoch) : locked.sessionEpoch
- const leaseEpoch = Math.max(locked.sessionEpoch + 1, previousLeaseEpoch + 1)
- if (existing.rows[0]) {
- await client.query(
- `UPDATE gfs_upload_parts
+ [locked.drive]
+ )
+ if (
+ Number(activeGlobal.rows[0]?.count ?? 0) >=
+ this.deps.config.maxConcurrentPartStreamsGlobal
+ ) {
+ throw new GfsError('quota_exceeded', 'global upload stream concurrency limit reached')
+ }
+ if (locked.activePartCount >= this.deps.config.maxConcurrentPartsPerSession)
+ throw new GfsError('quota_exceeded', 'per-session part concurrency limit reached')
+ const stagingPath = sessionPath(
+ this.deps.storageMountPath,
+ uploadId,
+ part.partNumber,
+ `.tmp-${randomUUID()}`
+ )
+ const previousLeaseEpoch = existing.rows[0]
+ ? Number(existing.rows[0].lease_epoch)
+ : locked.sessionEpoch
+ const leaseEpoch = Math.max(locked.sessionEpoch + 1, previousLeaseEpoch + 1)
+ if (existing.rows[0]) {
+ await client.query(
+ `UPDATE gfs_upload_parts
SET offset_bytes = $3, length_bytes = $4, sha256 = $5, state = 'reserved',
staging_path = $6, lease_epoch = $7, lease_started_at = now(), committed_at = NULL
WHERE upload_id = $1 AND part_number = $2`,
- [uploadId, part.partNumber, part.offsetBytes, part.lengthBytes, expectedChecksum, stagingPath, leaseEpoch]
- )
- } else {
- await client.query(
- `INSERT INTO gfs_upload_parts
+ [
+ uploadId,
+ part.partNumber,
+ part.offsetBytes,
+ part.lengthBytes,
+ expectedChecksum,
+ stagingPath,
+ leaseEpoch,
+ ]
+ )
+ } else {
+ await client.query(
+ `INSERT INTO gfs_upload_parts
(upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch, lease_started_at)
VALUES ($1,$2,$3,$4,$5,'reserved',$6,$7,now())`,
- [uploadId, part.partNumber, part.offsetBytes, part.lengthBytes, expectedChecksum, stagingPath, leaseEpoch]
- )
- }
- await client.query(
- `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = active_part_count + 1, updated_at = now()
+ [
+ uploadId,
+ part.partNumber,
+ part.offsetBytes,
+ part.lengthBytes,
+ expectedChecksum,
+ stagingPath,
+ leaseEpoch,
+ ]
+ )
+ }
+ await client.query(
+ `UPDATE gfs_upload_sessions SET state = 'uploading', active_part_count = active_part_count + 1, updated_at = now()
WHERE upload_id = $1`,
- [uploadId]
- )
- return { session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 }, existing: null as UploadPartRow | null, stagingPath, leaseEpoch }
+ [uploadId]
+ )
+ return {
+ session: { ...locked, state: 'uploading', activePartCount: locked.activePartCount + 1 },
+ existing: null as UploadPartRow | null,
+ stagingPath,
+ leaseEpoch,
+ }
})
} catch (error) {
this.inFlight.delete(key)
@@ -723,18 +869,31 @@ export class GfsUploadSessionService {
)
const postReplayRow = postReplay.rows[0]
if (
- this.canceledUploads.has(uploadId)
- || !postReplayRow
- || String(postReplayRow.state) === 'aborted'
- || Number(postReplayRow.session_epoch) !== reserved.session.sessionEpoch
+ this.canceledUploads.has(uploadId) ||
+ !postReplayRow ||
+ String(postReplayRow.state) === 'aborted' ||
+ Number(postReplayRow.session_epoch) !== reserved.session.sessionEpoch
) {
- controller.abort(new GfsError('upload_aborted', 'upload session changed while replay was starting'))
+ controller.abort(
+ new GfsError('upload_aborted', 'upload session changed while replay was starting')
+ )
}
- const observed = await readPartBody(req, reserved.existing.lengthBytes, this.deps.config.maxPartBytes, undefined, controller.signal)
- if (observed.sha256 !== reserved.existing.sha256 || observed.bytes !== reserved.existing.lengthBytes || expectedChecksum !== reserved.existing.sha256) {
+ const observed = await readPartBody(
+ req,
+ reserved.existing.lengthBytes,
+ this.deps.config.maxPartBytes,
+ undefined,
+ controller.signal
+ )
+ if (
+ observed.sha256 !== reserved.existing.sha256 ||
+ observed.bytes !== reserved.existing.lengthBytes ||
+ expectedChecksum !== reserved.existing.sha256
+ ) {
throw new GfsError('conflict', 'committed part checksum or length differs')
}
- if (this.canceledUploads.has(uploadId)) throw new GfsError('upload_aborted', 'upload session was canceled')
+ if (this.canceledUploads.has(uploadId))
+ throw new GfsError('upload_aborted', 'upload session was canceled')
return { part: reserved.existing, session: toReceipt(reserved.session) }
} finally {
clearTimeout(replayTimer)
@@ -756,15 +915,24 @@ export class GfsUploadSessionService {
)
const postReservationRow = postReservation.rows[0]
if (
- this.canceledUploads.has(uploadId)
- || !postReservationRow
- || String(postReservationRow.state) === 'aborted'
- || Number(postReservationRow.session_epoch) !== reserved.session.sessionEpoch
+ this.canceledUploads.has(uploadId) ||
+ !postReservationRow ||
+ String(postReservationRow.state) === 'aborted' ||
+ Number(postReservationRow.session_epoch) !== reserved.session.sessionEpoch
) {
- controller.abort(new GfsError('upload_aborted', 'upload session changed while part was reserving'))
+ controller.abort(
+ new GfsError('upload_aborted', 'upload session changed while part was reserving')
+ )
}
- const observed = await readPartBody(req, part.lengthBytes, this.deps.config.maxPartBytes, stagingPath, controller.signal)
- if (observed.sha256 !== expectedChecksum) throw new GfsError('checksum_mismatch', 'part checksum does not match Upload-Checksum')
+ const observed = await readPartBody(
+ req,
+ part.lengthBytes,
+ this.deps.config.maxPartBytes,
+ stagingPath,
+ controller.signal
+ )
+ if (observed.sha256 !== expectedChecksum)
+ throw new GfsError('checksum_mismatch', 'part checksum does not match Upload-Checksum')
finalPath = sessionPath(this.deps.storageMountPath, uploadId, part.partNumber, '')
try {
await stat(finalPath)
@@ -782,7 +950,13 @@ export class GfsUploadSessionService {
[uploadId, part.partNumber]
)
const row = rowResult.rows[0] ? rowToPart(rowResult.rows[0]) : null
- if (!row || row.state !== 'reserved' || row.leaseEpoch !== reserved.leaseEpoch || locked.sessionEpoch !== reserved.session.sessionEpoch || locked.state !== 'uploading') {
+ if (
+ !row ||
+ row.state !== 'reserved' ||
+ row.leaseEpoch !== reserved.leaseEpoch ||
+ locked.sessionEpoch !== reserved.session.sessionEpoch ||
+ locked.state !== 'uploading'
+ ) {
throw new GfsError('upload_aborted', 'upload session changed while part was in flight')
}
await client.query(
@@ -836,21 +1010,31 @@ export class GfsUploadSessionService {
reconciliation = undefined
}
}
- if (reconciliation) return { part: reconciliation.part, session: toReceipt(reconciliation.session) }
+ if (reconciliation)
+ return { part: reconciliation.part, session: toReceipt(reconciliation.session) }
await removeIfPresent(stagingPath)
if (finalPath && reconciliation !== undefined) await removeIfPresent(finalPath)
- await this.deps.tx.transaction(async client => {
- await client.query(
- `UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL
- WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'`,
- [uploadId, part.partNumber]
- )
- await client.query(
- `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()
- WHERE upload_id = $1`,
- [uploadId]
- )
- }).catch(() => undefined)
+ await this.deps.tx
+ .transaction(async client => {
+ const failedPart = await client.query(
+ `UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL
+ WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'
+ RETURNING 1`,
+ [uploadId, part.partNumber]
+ )
+ // The commit transaction already decremented the durable counter when
+ // it reached PostgreSQL. Only a part that was still reserved needs a
+ // recovery decrement; otherwise a lost response plus an inconclusive
+ // reconciliation would undercount other concurrent parts.
+ if (failedPart.rows.length > 0) {
+ await client.query(
+ `UPDATE gfs_upload_sessions SET active_part_count = GREATEST(active_part_count - 1, 0), updated_at = now()
+ WHERE upload_id = $1`,
+ [uploadId]
+ )
+ }
+ })
+ .catch(() => undefined)
throw error
} finally {
clearTimeout(partTimer)
@@ -861,10 +1045,18 @@ export class GfsUploadSessionService {
async pause(uploadId: string, principal: UploadPrincipal): Promise<UploadSessionReceipt> {
requireEnabled(this.deps.config)
const session = await this.loadOwned(uploadId, principal)
- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)
+ await this.deps.authorize?.(
+ principal,
+ session.operation,
+ session.operation === 'create'
+ ? (session.parentRid ?? undefined)
+ : (session.resourceRid ?? undefined)
+ )
if (session.state === 'paused') return toReceipt(session)
- if (session.state !== 'initiated' && session.state !== 'uploading') throw new GfsError('conflict', `cannot pause upload in ${session.state}`)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is already pausing')
+ if (session.state !== 'initiated' && session.state !== 'uploading')
+ throw new GfsError('conflict', `cannot pause upload in ${session.state}`)
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is already pausing')
this.pauseRequested.add(uploadId)
try {
const deadline = this.now() + this.deps.config.partTimeoutMs
@@ -883,13 +1075,22 @@ export class GfsUploadSessionService {
throw new GfsError('conflict', `cannot pause upload in ${locked.state}`)
}
if (locked.activePartCount !== 0) return locked
- await client.query(`UPDATE gfs_upload_sessions SET state = 'paused', updated_at = now() WHERE upload_id = $1 AND state IN ('initiated','uploading') AND active_part_count = 0`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET state = 'paused', updated_at = now() WHERE upload_id = $1 AND state IN ('initiated','uploading') AND active_part_count = 0`,
+ [uploadId]
+ )
return { ...locked, state: 'paused' }
})
if (paused.state === 'paused') return toReceipt(paused)
}
- if (this.now() >= deadline) throw new GfsError('precondition_failed', 'upload pause timed out while draining in-flight parts')
- await new Promise<void>(resolve => setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now()))))
+ if (this.now() >= deadline)
+ throw new GfsError(
+ 'precondition_failed',
+ 'upload pause timed out while draining in-flight parts'
+ )
+ await new Promise<void>(resolve =>
+ setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now())))
+ )
}
} finally {
this.pauseRequested.delete(uploadId)
@@ -900,13 +1101,24 @@ export class GfsUploadSessionService {
requireEnabled(this.deps.config)
const session = await this.loadOwned(uploadId, principal)
assertNotExpired(session, this.now())
- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')
+ await this.deps.authorize?.(
+ principal,
+ session.operation,
+ session.operation === 'create'
+ ? (session.parentRid ?? undefined)
+ : (session.resourceRid ?? undefined)
+ )
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is pausing')
if (session.state === 'uploading' || session.state === 'initiated') return toReceipt(session)
- if (session.state !== 'paused') throw new GfsError('conflict', `cannot resume upload in ${session.state}`)
+ if (session.state !== 'paused')
+ throw new GfsError('conflict', `cannot resume upload in ${session.state}`)
const resumed = await this.deps.tx.transaction(async client => {
const locked = await this.lockSession(client, uploadId, principal)
- await client.query(`UPDATE gfs_upload_sessions SET state = 'uploading', updated_at = now() WHERE upload_id = $1 AND state = 'paused'`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET state = 'uploading', updated_at = now() WHERE upload_id = $1 AND state = 'paused'`,
+ [uploadId]
+ )
return { ...locked, state: 'uploading' }
})
return toReceipt(resumed)
@@ -915,30 +1127,47 @@ export class GfsUploadSessionService {
async cancel(uploadId: string, principal: UploadPrincipal): Promise<void> {
requireEnabled(this.deps.config)
const session = await this.loadOwned(uploadId, principal)
- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)
- if (session.state === 'completed') throw new GfsError('upload_completed', 'completed upload cannot be canceled')
- if (session.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is in progress')
+ await this.deps.authorize?.(
+ principal,
+ session.operation,
+ session.operation === 'create'
+ ? (session.parentRid ?? undefined)
+ : (session.resourceRid ?? undefined)
+ )
+ if (session.state === 'completed')
+ throw new GfsError('upload_completed', 'completed upload cannot be canceled')
+ if (session.state === 'finalizing')
+ throw new GfsError('upload_finalizing', 'upload finalization is in progress')
if (session.state === 'aborted') return
this.canceledUploads.add(uploadId)
try {
await this.deps.tx.transaction(async client => {
const locked = await this.lockSession(client, uploadId, principal)
- if (locked.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is in progress')
- if (locked.state === 'completed') throw new GfsError('upload_completed', 'completed upload cannot be canceled')
+ if (locked.state === 'finalizing')
+ throw new GfsError('upload_finalizing', 'upload finalization is in progress')
+ if (locked.state === 'completed')
+ throw new GfsError('upload_completed', 'completed upload cannot be canceled')
// Once the row is fenced as aborted no new reservation can enter. Any
// reserved rows are now owned by the cancel drain (or the startup
// reconciler), so the durable counter must not keep DELETE waiting for
// a process that may have crashed before registering an in-flight
// controller.
- await client.query(`UPDATE gfs_upload_sessions SET state = 'aborted', session_epoch = session_epoch + 1, active_part_count = 0, updated_at = now() WHERE upload_id = $1`, [uploadId])
- await client.query(`UPDATE gfs_upload_parts SET state = 'failed' WHERE upload_id = $1 AND state = 'reserved'`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET state = 'aborted', session_epoch = session_epoch + 1, active_part_count = 0, updated_at = now() WHERE upload_id = $1`,
+ [uploadId]
+ )
+ await client.query(
+ `UPDATE gfs_upload_parts SET state = 'failed' WHERE upload_id = $1 AND state = 'reserved'`,
+ [uploadId]
+ )
})
} catch (error) {
this.canceledUploads.delete(uploadId)
throw error
}
for (const [key, flight] of this.inFlight) {
- if (key.startsWith(`${uploadId}:`)) flight.controller.abort(new GfsError('upload_aborted', 'upload canceled'))
+ if (key.startsWith(`${uploadId}:`))
+ flight.controller.abort(new GfsError('upload_aborted', 'upload canceled'))
}
await this.waitForActivePartsZero(uploadId, principal)
await this.deps.tx.transaction(async client => {
@@ -948,7 +1177,10 @@ export class GfsUploadSessionService {
if (cleaned) {
await this.deps.tx.transaction(async client => {
await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'aborted'`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'aborted'`,
+ [uploadId]
+ )
})
}
this.canceledUploads.delete(uploadId)
@@ -975,13 +1207,24 @@ export class GfsUploadSessionService {
for (const row of stale.rows) {
const uploadId = String(row.upload_id)
const partNumber = Number(row.part_number)
- const safePath = await validatedTempPath(this.deps.storageMountPath, uploadId, partNumber, String(row.staging_path))
+ const safePath = await validatedTempPath(
+ this.deps.storageMountPath,
+ uploadId,
+ partNumber,
+ String(row.staging_path)
+ )
if (!safePath) continue
// The row lock fences a new lease for this part. Remove both the
// recorded temporary file and a possible rename-before-commit orphan
// while the lock is still held; deleting the common final path after
// releasing the row would race a fresh reservation.
- if (!(await removeStrict(safePath) && await removeStrict(sessionPath(this.deps.storageMountPath, uploadId, partNumber, '')))) continue
+ if (
+ !(
+ (await removeStrict(safePath)) &&
+ (await removeStrict(sessionPath(this.deps.storageMountPath, uploadId, partNumber, '')))
+ )
+ )
+ continue
const failed = await client.query(
`UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL
WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'
@@ -1065,98 +1308,171 @@ export class GfsUploadSessionService {
})
for (const [uploadId, cleanup] of uploadDirs) {
for (const [key, flight] of this.inFlight) {
- if (key.startsWith(`${uploadId}:`)) flight.controller.abort(new GfsError('upload_aborted', 'upload session expired or was canceled'))
+ if (key.startsWith(`${uploadId}:`))
+ flight.controller.abort(
+ new GfsError('upload_aborted', 'upload session expired or was canceled')
+ )
}
- this.inFlightFinalizers.get(uploadId)?.controller.abort(
- new GfsError('upload_aborted', 'upload session expired or was canceled')
- )
+ this.inFlightFinalizers
+ .get(uploadId)
+ ?.controller.abort(new GfsError('upload_aborted', 'upload session expired or was canceled'))
const deadline = this.now() + this.deps.config.partTimeoutMs
while (this.hasInFlight(uploadId) && this.now() < deadline) {
await new Promise<void>(resolve => setTimeout(resolve, 25))
}
- if (!this.hasInFlight(uploadId) && await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {
- await this.deps.tx.transaction(async client => {
- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
- if (cleanup.purgeReceipt) {
- await client.query(`DELETE FROM gfs_upload_sessions WHERE upload_id = $1 AND state = 'completed'`, [uploadId])
- } else {
- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed')`, [uploadId])
- }
- }).catch(() => undefined)
+ if (
+ !this.hasInFlight(uploadId) &&
+ (await removeUploadDirectory(this.deps.storageMountPath, uploadId))
+ ) {
+ await this.deps.tx
+ .transaction(async client => {
+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
+ if (cleanup.purgeReceipt) {
+ await client.query(
+ `DELETE FROM gfs_upload_sessions WHERE upload_id = $1 AND state = 'completed'`,
+ [uploadId]
+ )
+ } else {
+ await client.query(
+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed')`,
+ [uploadId]
+ )
+ }
+ })
+ .catch(() => undefined)
this.canceledUploads.delete(uploadId)
} else {
// Rotate failed cleanup attempts so a permanently broken oldest batch
// cannot starve newer terminal sessions. The terminal row remains
// eligible (`cleanup_at IS NULL`) and will be retried next cycle.
- await this.deps.tx.transaction(async client => {
- await client.query(
- `UPDATE gfs_upload_sessions SET updated_at = now()
+ await this.deps.tx
+ .transaction(async client => {
+ await client.query(
+ `UPDATE gfs_upload_sessions SET updated_at = now()
WHERE upload_id = $1 AND state IN ('aborted','expired','failed','completed') AND cleanup_at IS NULL`,
- [uploadId]
- )
- }).catch(() => undefined)
+ [uploadId]
+ )
+ })
+ .catch(() => undefined)
}
}
return { staleParts, expiredSessions }
}
- async complete(uploadId: string, principal: UploadPrincipal, wholeSha256?: string | null): Promise<UploadSessionReceipt> {
+ async complete(
+ uploadId: string,
+ principal: UploadPrincipal,
+ wholeSha256?: string | null
+ ): Promise<UploadSessionReceipt> {
requireEnabled(this.deps.config)
const session = await this.loadOwned(uploadId, principal)
assertNotExpired(session, this.now())
- await this.deps.authorize?.(principal, session.operation, session.operation === 'create' ? session.parentRid ?? undefined : session.resourceRid ?? undefined)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')
+ await this.deps.authorize?.(
+ principal,
+ session.operation,
+ session.operation === 'create'
+ ? (session.parentRid ?? undefined)
+ : (session.resourceRid ?? undefined)
+ )
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is pausing')
if (session.state === 'completed') return toReceipt(session)
- if (session.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is already in progress')
- if (!this.deps.finalize) throw new GfsError('not_mounted', 'upload finalization is not available yet')
- const expectedWhole = wholeSha256 === null || wholeSha256 === undefined
- ? session.wholeSha256
- : requireChecksum(wholeSha256, 'wholeSha256')
- const started: UploadSessionRow & { parts?: UploadPartRow[] } = await this.deps.tx.transaction(async client => {
- const locked = await this.lockSession(client, uploadId, principal)
- if (this.pauseRequested.has(uploadId)) throw new GfsError('conflict', 'upload session is pausing')
- if (locked.state === 'completed') return locked
- if (locked.state === 'finalizing') throw new GfsError('upload_finalizing', 'upload finalization is already in progress')
- if (!['initiated', 'uploading', 'paused'].includes(locked.state)) {
- if (locked.state === 'aborted') throw new GfsError('upload_aborted', 'upload session was canceled')
- if (locked.state === 'expired') throw new GfsError('upload_expired', 'upload session has expired')
- throw new GfsError('conflict', `cannot complete upload in ${locked.state}`)
- }
- if (locked.wholeSha256 && expectedWhole && locked.wholeSha256 !== expectedWhole) {
- throw new GfsError('conflict', 'whole-file checksum differs from the session checksum')
- }
- const parts = await client.query(`SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' ORDER BY part_number`, [uploadId])
- const committedParts = parts.rows.map(rowToPart)
- if (committedParts.length !== locked.partCount) throw new GfsError('upload_incomplete', 'not all upload parts are committed')
- for (let index = 0; index < locked.partCount; index += 1) {
- const actual = committedParts[index]
- const expected = partGeometry({ expectedBytes: locked.expectedBytes, partBytes: locked.partBytes }, index)
- if (!actual || actual.partNumber !== expected.partNumber || actual.offsetBytes !== expected.offsetBytes || actual.lengthBytes !== expected.lengthBytes) {
- throw new GfsError('upload_incomplete', `upload part ${index} does not match the session geometry`)
+ if (session.state === 'finalizing')
+ throw new GfsError('upload_finalizing', 'upload finalization is already in progress')
+ if (!this.deps.finalize)
+ throw new GfsError('not_mounted', 'upload finalization is not available yet')
+ const expectedWhole =
+ wholeSha256 === null || wholeSha256 === undefined
+ ? session.wholeSha256
+ : requireChecksum(wholeSha256, 'wholeSha256')
+ const started: UploadSessionRow & { parts?: UploadPartRow[] } = await this.deps.tx.transaction(
+ async client => {
+ const locked = await this.lockSession(client, uploadId, principal)
+ if (this.pauseRequested.has(uploadId))
+ throw new GfsError('conflict', 'upload session is pausing')
+ if (locked.state === 'completed') return locked
+ if (locked.state === 'finalizing')
+ throw new GfsError('upload_finalizing', 'upload finalization is already in progress')
+ if (!['initiated', 'uploading', 'paused'].includes(locked.state)) {
+ if (locked.state === 'aborted')
+ throw new GfsError('upload_aborted', 'upload session was canceled')
+ if (locked.state === 'expired')
+ throw new GfsError('upload_expired', 'upload session has expired')
+ throw new GfsError('conflict', `cannot complete upload in ${locked.state}`)
+ }
+ if (locked.wholeSha256 && expectedWhole && locked.wholeSha256 !== expectedWhole) {
+ throw new GfsError('conflict', 'whole-file checksum differs from the session checksum')
+ }
+ const parts = await client.query(
+ `SELECT upload_id, part_number, offset_bytes, length_bytes, sha256, state, staging_path, lease_epoch FROM gfs_upload_parts WHERE upload_id = $1 AND state = 'committed' ORDER BY part_number`,
+ [uploadId]
+ )
+ const committedParts = parts.rows.map(rowToPart)
+ if (committedParts.length !== locked.partCount)
+ throw new GfsError('upload_incomplete', 'not all upload parts are committed')
+ for (let index = 0; index < locked.partCount; index += 1) {
+ const actual = committedParts[index]
+ const expected = partGeometry(
+ { expectedBytes: locked.expectedBytes, partBytes: locked.partBytes },
+ index
+ )
+ if (
+ !actual ||
+ actual.partNumber !== expected.partNumber ||
+ actual.offsetBytes !== expected.offsetBytes ||
+ actual.lengthBytes !== expected.lengthBytes
+ ) {
+ throw new GfsError(
+ 'upload_incomplete',
+ `upload part ${index} does not match the session geometry`
+ )
+ }
+ }
+ if (
+ locked.activePartCount !== 0 ||
+ locked.committedPartCount !== locked.partCount ||
+ locked.committedBytes !== locked.expectedBytes
+ )
+ throw new GfsError('upload_incomplete', 'not all upload parts are committed')
+ await client.query(
+ `SELECT pg_advisory_xact_lock(hashtext('gfs:upload:finalizers')::bigint)`,
+ []
+ )
+ const finalizing = await client.query(
+ `SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'`,
+ []
+ )
+ if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {
+ throw new GfsError('quota_exceeded', 'upload finalization concurrency limit reached')
+ }
+ if (!locked.wholeSha256 && expectedWhole) {
+ await client.query(
+ `UPDATE gfs_upload_sessions SET whole_sha256 = $2, updated_at = now() WHERE upload_id = $1`,
+ [uploadId, expectedWhole]
+ )
+ }
+ await client.query(
+ `UPDATE gfs_upload_sessions SET state = 'finalizing', updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`,
+ [uploadId, locked.sessionEpoch]
+ )
+ return {
+ ...locked,
+ wholeSha256: expectedWhole ?? locked.wholeSha256,
+ state: 'finalizing' as const,
+ parts: committedParts,
}
}
- if (locked.activePartCount !== 0 || locked.committedPartCount !== locked.partCount || locked.committedBytes !== locked.expectedBytes) throw new GfsError('upload_incomplete', 'not all upload parts are committed')
- await client.query(`SELECT pg_advisory_xact_lock(hashtext('gfs:upload:finalizers')::bigint)`, [])
- const finalizing = await client.query(
- `SELECT COUNT(*)::bigint AS count FROM gfs_upload_sessions WHERE state = 'finalizing'`,
- []
- )
- if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {
- throw new GfsError('quota_exceeded', 'upload finalization concurrency limit reached')
- }
- if (!locked.wholeSha256 && expectedWhole) {
- await client.query(`UPDATE gfs_upload_sessions SET whole_sha256 = $2, updated_at = now() WHERE upload_id = $1`, [uploadId, expectedWhole])
- }
- await client.query(`UPDATE gfs_upload_sessions SET state = 'finalizing', updated_at = now() WHERE upload_id = $1 AND session_epoch = $2`, [uploadId, locked.sessionEpoch])
- return { ...locked, wholeSha256: expectedWhole ?? locked.wholeSha256, state: 'finalizing' as const, parts: committedParts }
- })
+ )
if (started.state === 'completed') return toReceipt(started)
if (!started.parts) throw new GfsError('internal', 'finalization parts were not loaded')
let receipt: { resourceId: string; version: number; sha256: string }
const finalizationAbort = new AbortController()
const finalizationDeadlineAtMs = this.now() + this.deps.config.finalizeTimeoutMs
const finalizationTimer = setTimeout(
- () => finalizationAbort.abort(new GfsError('precondition_failed', 'upload finalization timed out')),
+ () =>
+ finalizationAbort.abort(
+ new GfsError('precondition_failed', 'upload finalization timed out')
+ ),
Math.max(1, this.deps.config.finalizeTimeoutMs)
)
finalizationTimer.unref?.()
@@ -1164,9 +1480,17 @@ export class GfsUploadSessionService {
clearTimeout(finalizationTimer)
throw new GfsError('upload_finalizing', 'upload finalization is already in progress')
}
- this.inFlightFinalizers.set(uploadId, { controller: finalizationAbort, sessionEpoch: started.sessionEpoch })
+ this.inFlightFinalizers.set(uploadId, {
+ controller: finalizationAbort,
+ sessionEpoch: started.sessionEpoch,
+ })
try {
- receipt = await this.deps.finalize(started, started.parts, finalizationAbort.signal, finalizationDeadlineAtMs)
+ receipt = await this.deps.finalize(
+ started,
+ started.parts,
+ finalizationAbort.signal,
+ finalizationDeadlineAtMs
+ )
} catch (error) {
const failureCode = error instanceof GfsError ? error.code : 'finalization_failed'
await this.deps.tx.transaction(async client => {
@@ -1187,11 +1511,19 @@ export class GfsUploadSessionService {
const current = this.inFlightFinalizers.get(uploadId)
if (current?.controller === finalizationAbort) this.inFlightFinalizers.delete(uploadId)
await this.waitForQuiescence(uploadId)
- if (!this.hasInFlight(uploadId) && await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {
- await this.deps.tx.transaction(async client => {
- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`, [uploadId])
- }).catch(() => undefined)
+ if (
+ !this.hasInFlight(uploadId) &&
+ (await removeUploadDirectory(this.deps.storageMountPath, uploadId))
+ ) {
+ await this.deps.tx
+ .transaction(async client => {
+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'failed'`,
+ [uploadId]
+ )
+ })
+ .catch(() => undefined)
}
throw error
} finally {
@@ -1204,25 +1536,35 @@ export class GfsUploadSessionService {
completed = await this.deps.tx.transaction(async client => {
const locked = await this.lockSession(client, uploadId, principal)
if (locked.state === 'completed') return locked
- if (locked.state !== 'finalizing' || locked.sessionEpoch !== started.sessionEpoch) throw new GfsError('upload_aborted', 'upload session changed during finalization')
+ if (locked.state !== 'finalizing' || locked.sessionEpoch !== started.sessionEpoch)
+ throw new GfsError('upload_aborted', 'upload session changed during finalization')
await client.query(
`UPDATE gfs_upload_sessions SET state = 'completed', result_resource_id = $2, result_version = $3, result_sha256 = $4, completed_at = now(), updated_at = now() WHERE upload_id = $1`,
[uploadId, receipt.resourceId, receipt.version, receipt.sha256]
)
- return { ...locked, state: 'completed', resultResourceId: receipt.resourceId, resultVersion: receipt.version, resultSha256: receipt.sha256 }
+ return {
+ ...locked,
+ state: 'completed',
+ resultResourceId: receipt.resourceId,
+ resultVersion: receipt.version,
+ resultSha256: receipt.sha256,
+ }
})
} catch (error) {
- if (error instanceof CommitOutcomeUnknownError || error instanceof RollbackOutcomeUnknownError) {
+ if (
+ error instanceof CommitOutcomeUnknownError ||
+ error instanceof RollbackOutcomeUnknownError
+ ) {
const observed = await this.deps.db.query(
`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,
[uploadId, principal.drive, principal.ownerSubject]
)
const row = observed.rows[0] ? rowToSession(observed.rows[0]) : null
if (
- row?.state === 'completed'
- && row.resultResourceId === receipt.resourceId
- && row.resultVersion === receipt.version
- && row.resultSha256 === receipt.sha256
+ row?.state === 'completed' &&
+ row.resultResourceId === receipt.resourceId &&
+ row.resultVersion === receipt.version &&
+ row.resultSha256 === receipt.sha256
) {
completed = row
} else {
@@ -1233,24 +1575,35 @@ export class GfsUploadSessionService {
}
}
if (await removeUploadDirectory(this.deps.storageMountPath, uploadId)) {
- await this.deps.tx.transaction(async client => {
- await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
- await client.query(`UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'completed'`, [uploadId])
- }).catch(() => undefined)
+ await this.deps.tx
+ .transaction(async client => {
+ await client.query(`DELETE FROM gfs_upload_parts WHERE upload_id = $1`, [uploadId])
+ await client.query(
+ `UPDATE gfs_upload_sessions SET cleanup_at = now(), updated_at = now() WHERE upload_id = $1 AND state = 'completed'`,
+ [uploadId]
+ )
+ })
+ .catch(() => undefined)
}
return toReceipt(completed)
}
private async loadOwned(uploadId: string, principal: UploadPrincipal): Promise<UploadSessionRow> {
requireUuid(uploadId, 'uploadId')
- const result = await this.deps.db.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`, [uploadId, principal.drive, principal.ownerSubject])
+ const result = await this.deps.db.query(
+ `SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,
+ [uploadId, principal.drive, principal.ownerSubject]
+ )
if (!result.rows[0]) throw new GfsError('not_found', 'upload session not found')
const session = rowToSession(result.rows[0])
assertNotExpired(session, this.now())
return session
}
- private async waitForActivePartsZero(uploadId: string, principal: UploadPrincipal): Promise<void> {
+ private async waitForActivePartsZero(
+ uploadId: string,
+ principal: UploadPrincipal
+ ): Promise<void> {
const deadline = this.now() + this.deps.config.partTimeoutMs
for (;;) {
const result = await this.deps.db.query(
@@ -1261,8 +1614,14 @@ export class GfsUploadSessionService {
if (!row) throw new GfsError('not_found', 'upload session not found')
const localInFlight = this.hasInFlight(uploadId)
if (row.activePartCount === 0 && !localInFlight) return
- if (this.now() >= deadline) throw new GfsError('precondition_failed', 'upload cancellation timed out while draining in-flight parts')
- await new Promise<void>(resolve => setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now()))))
+ if (this.now() >= deadline)
+ throw new GfsError(
+ 'precondition_failed',
+ 'upload cancellation timed out while draining in-flight parts'
+ )
+ await new Promise<void>(resolve =>
+ setTimeout(resolve, Math.min(25, Math.max(1, deadline - this.now())))
+ )
}
}
@@ -1289,13 +1648,14 @@ export class GfsUploadSessionService {
if (!result.rows[0]) return null
const committedPart = rowToPart(result.rows[0])
if (
- committedPart.state !== 'committed'
- || committedPart.sha256 !== checksum
- || committedPart.offsetBytes !== part.offsetBytes
- || committedPart.lengthBytes !== part.lengthBytes
- || committedPart.leaseEpoch !== leaseEpoch
- || committedPart.stagingPath !== finalPath
- ) return null
+ committedPart.state !== 'committed' ||
+ committedPart.sha256 !== checksum ||
+ committedPart.offsetBytes !== part.offsetBytes ||
+ committedPart.lengthBytes !== part.lengthBytes ||
+ committedPart.leaseEpoch !== leaseEpoch ||
+ committedPart.stagingPath !== finalPath
+ )
+ return null
const sessionResult = await this.deps.db.query(
`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3`,
[uploadId, principal.drive, principal.ownerSubject]
@@ -1306,8 +1666,15 @@ export class GfsUploadSessionService {
return { part: committedPart, session }
}
- private async lockSession(client: TxClient, uploadId: string, principal: UploadPrincipal): Promise<UploadSessionRow> {
- const result = await client.query(`SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3 FOR UPDATE`, [uploadId, principal.drive, principal.ownerSubject])
+ private async lockSession(
+ client: TxClient,
+ uploadId: string,
+ principal: UploadPrincipal
+ ): Promise<UploadSessionRow> {
+ const result = await client.query(
+ `SELECT * FROM gfs_upload_sessions WHERE upload_id = $1 AND drive = $2 AND owner_subject = $3 FOR UPDATE`,
+ [uploadId, principal.drive, principal.ownerSubject]
+ )
if (!result.rows[0]) throw new GfsError('not_found', 'upload session not found')
return rowToSession(result.rows[0])
}
@@ -1318,23 +1685,32 @@ export const GFS_UPLOAD_REQUEST_BODY_LIMITS = {
completeBytes: GFS_UPLOAD_V2_COMPLETE_BODY_MAX_BYTES,
}
-export function uploadCapabilities(enabled: boolean, config: GfsUploadConfig): {
+export function uploadCapabilities(
+ enabled: boolean,
+ config: GfsUploadConfig
+): {
legacyBase64: { enabled: true; maxFileBytes: number }
- resumableV2: ReturnType<typeof disabledGfsUploadV2Capability> | {
- enabled: true
- maxFileBytes: number
- preferredChunkBytes: number
- maxChunkBytes: number
- maxConcurrentPartsPerSession: number
- fallbackConcurrency: number
- maxConcurrentPartStreamsGlobal: number
- instabilityFailureThreshold: number
- sessionTtlSeconds: number
- checksumAlgorithms: ['sha256']
- wholeFileChecksumRequired: boolean
- }
+ resumableV2:
+ | ReturnType<typeof disabledGfsUploadV2Capability>
+ | {
+ enabled: true
+ maxFileBytes: number
+ preferredChunkBytes: number
+ maxChunkBytes: number
+ maxConcurrentPartsPerSession: number
+ fallbackConcurrency: number
+ maxConcurrentPartStreamsGlobal: number
+ instabilityFailureThreshold: number
+ sessionTtlSeconds: number
+ checksumAlgorithms: ['sha256']
+ wholeFileChecksumRequired: boolean
+ }
} {
- if (!enabled) return { legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 }, resumableV2: disabledGfsUploadV2Capability() }
+ if (!enabled)
+ return {
+ legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 },
+ resumableV2: disabledGfsUploadV2Capability(),
+ }
return {
legacyBase64: { enabled: true, maxFileBytes: 16 * 1024 * 1024 },
resumableV2: {
diff --git a/scripts/tests/test-gfs-deploy-contract.sh b/scripts/tests/test-gfs-deploy-contract.sh
index 7a3525759..d7c49b88c 100755
--- a/scripts/tests/test-gfs-deploy-contract.sh
+++ b/scripts/tests/test-gfs-deploy-contract.sh
@@ -2,6 +2,19 @@
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"; cd "$ROOT"
fail() { echo "FAIL: $*" >&2; exit 1; }
+external_rest_manifest="$(cat deploy/base/profiles/external-rest-api.yaml)"
+profiles_network_policy="$(cat deploy/base/profiles/networkpolicies.yaml)"
+ingress_network_policy="$(cat deploy/base/ingress/networkpolicies.yaml)"
+[[ "$external_rest_manifest" == *'type: ClusterIP'* ]] \
+ || fail 'external-rest-api must remain an internal ClusterIP, not a directly exposed service'
+[[ "$profiles_network_policy" == *'name: allow-ingress-profiles'* && \
+ "$profiles_network_policy" == *'app: cloudflared'* && \
+ "$profiles_network_policy" == *'values: [external-rest-api, profile-ui]'* && \
+ "$profiles_network_policy" == *'name: external-rest-api-from-profile-ui'* ]] \
+ || fail 'profiles ingress policy does not document the trusted Cloudflare/profile-ui paths'
+[[ "$ingress_network_policy" == *'app: external-rest-api'* && \
+ "$ingress_network_policy" == *'kubernetes.io/metadata.name: profiles'* ]] \
+ || fail 'ingress namespace policy does not restrict cloudflared egress to profiles services'
# Scope: OSS-resident GFS deploy invariants only. The GCP deploy invariants
# (deploy-dev/deploy-prod workflow ordering, gcp-* Makefile targets, gcp-prod
# overlay render, and the HCC rollback contract) live in evenfire-infra after