Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): recover expired finalizing uploads safely

Alfredo Lopez2a6d0b26b1e4
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index d6369407a..bfa0d0a98 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.451",
+ "version": "0.1.452",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.451",
+ "version": "0.1.452",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index a9613931e..1459f79eb 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.451",
+ "version": "0.1.452",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.ts
index 33f2fea08..23a688c6c 100644
--- a/control-api/src/middleware/gfsUploadAdmission.ts
+++ b/control-api/src/middleware/gfsUploadAdmission.ts
@@ -112,6 +112,7 @@ function sendBackendUnavailable(res: Response): void {
function sendRateLimited(res: Response, limit: string, retryAfter: number, max: number): void {
res.setHeader('Retry-After', String(retryAfter))
res.setHeader('X-RateLimit-Limit', String(max))
+ res.setHeader('X-GFS-RateLimit-Scope', limit)
res.setHeader('X-RateLimit-Remaining', '0')
res.status(429).json({
error: 'gfs_upload_rate_limited',
diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.ts
index 4ce938444..36a335589 100644
--- a/control-api/src/routes/gfs/proxy.ts
+++ b/control-api/src/routes/gfs/proxy.ts
@@ -30,6 +30,7 @@ const PASSTHROUGH_RESPONSE_HEADERS = [
'retry-after',
'x-ratelimit-limit',
'x-ratelimit-remaining',
+ 'x-gfs-ratelimit-scope',
]
const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
const UPLOAD_PART_PATH = new RegExp(`^/v1/uploads/${UUID_PATH}/parts/[0-9]+(?:\\?|$)`, 'i')
diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.ts
index c38970a36..d073cc6ee 100644
--- a/control-api/test/gfsUploadAdmission.test.ts
+++ b/control-api/test/gfsUploadAdmission.test.ts
@@ -163,6 +163,8 @@ describe('GFS upload admission', () => {
expect(response.status).toBe(429)
expect(response.headers['retry-after']).toMatch(/^\d+$/)
+ expect(response.headers['x-ratelimit-limit']).toBe('120')
+ expect(response.headers['x-gfs-ratelimit-scope']).toBe('principal_requests')
expect(response.body).toMatchObject({
error: 'gfs_upload_rate_limited',
limit: 'principal_requests',
diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
index 6b4783180..5af683795 100644
--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts
@@ -836,6 +836,40 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {
expect(new Date(String(row.rows[0]?.expires_at)).getTime()).toBeGreaterThan(Date.now())
})
+ it('expires a finalizing session with a missing durable start timestamp after its TTL', async () => {
+ const created = await uploads.create({
+ drive,
+ ownerSubject: owner,
+ primarySubject: owner,
+ operation: 'create',
+ parentRid: 'b'.repeat(32),
+ name: 'finalizing-null-start.bin',
+ sizeBytes: 0,
+ idempotencyKey: randomUUID(),
+ })
+ await pool.query(
+ `UPDATE gfs_upload_sessions
+ SET state = 'finalizing',
+ finalizing_started_at = NULL,
+ expires_at = now() - interval '1 second'
+ WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+
+ const result = await uploads.reconcile()
+ expect(result.expiredSessions).toBeGreaterThanOrEqual(1)
+ const row = await pool.query(
+ `SELECT state, finalizing_started_at, active_part_count
+ FROM gfs_upload_sessions WHERE upload_id = $1`,
+ [created.session.uploadId]
+ )
+ expect(row.rows[0]).toEqual({
+ state: 'expired',
+ finalizing_started_at: null,
+ active_part_count: 0,
+ })
+ })
+
it('rejects cancel during finalization and leaves one completed receipt', async () => {
let finalizerStarted!: () => void
let releaseFinalizer!: () => void
diff --git a/control-api/test/routes.gfsProxy.test.ts b/control-api/test/routes.gfsProxy.test.ts
index 33abb5edd..7e8cded9f 100644
--- a/control-api/test/routes.gfsProxy.test.ts
+++ b/control-api/test/routes.gfsProxy.test.ts
@@ -208,7 +208,8 @@ describe('/api/v1/gfs/proxy', () => {
headers: {
'content-type': 'application/json',
'retry-after': '3',
- 'x-ratelimit-limit': 'active_part_streams_global',
+ 'x-ratelimit-limit': '4',
+ 'x-gfs-ratelimit-scope': 'active_part_streams_global',
'x-ratelimit-remaining': '0',
'upload-length': '209715200',
},
@@ -221,7 +222,8 @@ describe('/api/v1/gfs/proxy', () => {
expect(res.status).toBe(429)
expect(res.headers['retry-after']).toBe('3')
- expect(res.headers['x-ratelimit-limit']).toBe('active_part_streams_global')
+ expect(res.headers['x-ratelimit-limit']).toBe('4')
+ expect(res.headers['x-gfs-ratelimit-scope']).toBe('active_part_streams_global')
expect(res.headers['x-ratelimit-remaining']).toBe('0')
expect(res.headers['upload-length']).toBe('209715200')
})
diff --git a/control-ui/components/GfsBrowser.tsx b/control-ui/components/GfsBrowser.tsx
index e984c7280..c55ff8db8 100644
--- a/control-ui/components/GfsBrowser.tsx
+++ b/control-ui/components/GfsBrowser.tsx
@@ -449,17 +449,7 @@ export function GfsBrowser(): React.JSX.Element {
assertGfsFileUploadSize(file.size)
const name = await normalizeGfsResourceName(file.name)
const target = { operation: 'create' as const, parentRid: rid }
- const pending = readPendingGfsUpload()
- resumeUploadId =
- pending &&
- pending.fileName === file.name &&
- pending.fileSize === file.size &&
- pending.lastModified === file.lastModified &&
- pending.target.operation === target.operation &&
- pending.target.parentRid === target.parentRid &&
- pending.name === name
- ? pending.uploadId
- : undefined
+ resumeUploadId = matchingPendingResumeUploadId(file, target, name)
const job = createGfsUploadJob({
file,
name,
diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsx
index 2a2d74e1c..9718293b2 100644
--- a/control-ui/components/__tests__/GfsBrowser.test.tsx
+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx
@@ -316,6 +316,46 @@ describe('GfsBrowser', () => {
)
})
+ it('reuses a matching persisted session from the button upload path', async () => {
+ const rootId = '11111111-1111-1111-1111-111111111111'
+ const rootRid = '11111111111111111111111111111111'
+ const uploadId = '88888888-8888-4888-8888-888888888888'
+ const lastModified = 1_725_000_000_000
+ const fileName = 'button-resume.md'
+ window.localStorage.setItem(
+ 'evenfire:gfs-upload-v2:pending',
+ JSON.stringify({
+ uploadId,
+ fileName,
+ fileSize: 11,
+ lastModified,
+ target: { operation: 'create', parentRid: rootRid },
+ name: fileName,
+ })
+ )
+ mockApiGet.mockResolvedValue({ rootResourceId: rootId, items: [], nextCursor: null })
+ renderBrowser()
+ await screen.findByText('No resources are visible in this folder.')
+
+ fireEvent.click(screen.getByRole('button', { name: /upload file/i }))
+ const dialog = await screen.findByRole('dialog', { name: 'Upload file' })
+ fireEvent.change(within(dialog).getByLabelText('Choose file to upload'), {
+ target: { files: [new File(['button data'], fileName, { lastModified })] },
+ })
+ fireEvent.click(within(dialog).getByRole('button', { name: 'Upload' }))
+
+ await waitFor(() =>
+ expect(mockCreateGfsUploadJob).toHaveBeenCalledWith(
+ expect.objectContaining({
+ file: expect.any(File),
+ name: fileName,
+ resumeUploadId: uploadId,
+ target: { operation: 'create', parentRid: rootRid },
+ })
+ )
+ )
+ })
+
it('uploads dropped images and Markdown files into the current folder', async () => {
const rootId = '11111111-1111-1111-1111-111111111111'
const rootRid = '11111111111111111111111111111111'
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index 58ebe4a5b..af98647a2 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.427",
+ "version": "0.1.428",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.427",
+ "version": "0.1.428",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/llm-providers": "file:../packages/llm-providers",
diff --git a/control-ui/package.json b/control-ui/package.json
index 1d8336375..83f040783 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.427",
+ "version": "0.1.428",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index 8e512a40f..660d39ec3 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.76",
+ "version": "0.1.77",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.76",
+ "version": "0.1.77",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index 604223d6d..714f06ede 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.76",
+ "version": "0.1.77",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index b971ac89a..ba6348093 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -8,7 +8,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.7.0',
- externalRestApiVersion: '0.1.76',
+ externalRestApiVersion: '0.1.77',
rpcProxyVersion: '0.1.64',
desktopVersion: '0.7.0',
minimumDesktopVersion: '0.1.252',
diff --git a/external-rest-api/src/routes/gfs.ts b/external-rest-api/src/routes/gfs.ts
index 04967eebb..7c1bea9fc 100644
--- a/external-rest-api/src/routes/gfs.ts
+++ b/external-rest-api/src/routes/gfs.ts
@@ -48,6 +48,7 @@ const STREAM_HEADERS = [
'retry-after',
'x-ratelimit-limit',
'x-ratelimit-remaining',
+ 'x-gfs-ratelimit-scope',
]
function forwardControlApiError(error: unknown, res: Response, next: NextFunction): void {
@@ -97,6 +98,7 @@ export function createGfsRouter(options: GfsRouterOptions = {}): Router {
const retryAfterSeconds = Math.max(1, Math.ceil((resetAt - Date.now()) / 1000))
res.setHeader('Retry-After', String(retryAfterSeconds))
res.setHeader('X-RateLimit-Limit', String(info?.limit ?? edgeRequestLimit))
+ res.setHeader('X-GFS-RateLimit-Scope', 'public_edge_requests')
res.setHeader('X-RateLimit-Remaining', '0')
res.status(429).json({
error: 'gfs_upload_rate_limited',
diff --git a/external-rest-api/test/routes.gfs.test.ts b/external-rest-api/test/routes.gfs.test.ts
index 0973698b8..e105bc32d 100644
--- a/external-rest-api/test/routes.gfs.test.ts
+++ b/external-rest-api/test/routes.gfs.test.ts
@@ -70,6 +70,7 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
expect(second.status).toBe(429)
expect(second.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)
expect(second.headers['x-ratelimit-limit']).toBe('1')
+ expect(second.headers['x-gfs-ratelimit-scope']).toBe('public_edge_requests')
expect(second.headers['x-ratelimit-remaining']).toBe('0')
expect(second.body).toEqual({
error: 'gfs_upload_rate_limited',
@@ -261,7 +262,11 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
'rate limited',
429,
{ error: 'gfs_upload_rate_limited', limit: 'principal_bytes', retryAfterSeconds: 7 },
- new Headers({ 'retry-after': '7' })
+ new Headers({
+ 'retry-after': '7',
+ 'x-ratelimit-limit': '512',
+ 'x-gfs-ratelimit-scope': 'principal_bytes',
+ })
)
)
const response = await request(buildApp())
@@ -269,6 +274,8 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',
.set('authorization', 'Bearer sess-xyz')
expect(response.status).toBe(429)
expect(response.headers['retry-after']).toBe('7')
+ expect(response.headers['x-ratelimit-limit']).toBe('512')
+ expect(response.headers['x-gfs-ratelimit-scope']).toBe('principal_bytes')
expect(response.body).toEqual({
error: 'gfs_upload_rate_limited',
limit: 'principal_bytes',
diff --git a/gfs-controller/src/api/envelope.ts b/gfs-controller/src/api/envelope.ts
index 8980e6ac7..54281852e 100644
--- a/gfs-controller/src/api/envelope.ts
+++ b/gfs-controller/src/api/envelope.ts
@@ -43,9 +43,13 @@ export function fail(error: GfsError): ErrEnvelope {
* declared code/status; anything else is an `internal` (500) — failures are
* surfaced, never swallowed into a misleading success.
*/
-export function toResponse(error: unknown): { status: number; body: ErrEnvelope } {
+export function toResponse(error: unknown): {
+ status: number;
+ body: ErrEnvelope;
+ rateLimitLimit?: number;
+} {
if (error instanceof GfsError) {
- return { status: error.status, body: fail(error) };
+ return { status: error.status, body: fail(error), rateLimitLimit: error.rateLimitLimit };
}
// Some lower layers throw typed errors carrying a `.code` that matches the
// envelope vocabulary (auth, blob store). Honor it when present; otherwise
diff --git a/gfs-controller/src/api/errors.ts b/gfs-controller/src/api/errors.ts
index 498bdff27..3ba87c480 100644
--- a/gfs-controller/src/api/errors.ts
+++ b/gfs-controller/src/api/errors.ts
@@ -68,12 +68,13 @@ export class GfsError extends Error {
readonly reason?: string;
readonly retryAfterSeconds?: number;
readonly limit?: string;
+ readonly rateLimitLimit?: number;
constructor(
code: GfsErrorCode,
message: string,
reason?: string,
- details?: { retryAfterSeconds?: number; limit?: string },
+ details?: { retryAfterSeconds?: number; limit?: string; rateLimitLimit?: number },
) {
super(message);
this.name = "GfsError";
@@ -81,6 +82,7 @@ export class GfsError extends Error {
this.reason = reason;
this.retryAfterSeconds = details?.retryAfterSeconds;
this.limit = details?.limit;
+ this.rateLimitLimit = details?.rateLimitLimit;
}
get status(): number {
diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.ts
index 61d010187..70cdb7199 100644
--- a/gfs-controller/src/api/serve.test.ts
+++ b/gfs-controller/src/api/serve.test.ts
@@ -279,7 +279,7 @@ describe("GFS Upload v2 route contract", () => {
"quota_exceeded",
"global upload stream concurrency limit reached",
undefined,
- { retryAfterSeconds: 1, limit: "active_part_streams_global" },
+ { retryAfterSeconds: 1, limit: "active_part_streams_global", rateLimitLimit: 4 },
);
await run(
uploadDeps({
@@ -295,7 +295,8 @@ describe("GFS Upload v2 route contract", () => {
);
expect(res.statusCode).toBe(429);
expect(res.headers["Retry-After"]).toBe("1");
- expect(res.headers["X-RateLimit-Limit"]).toBe("active_part_streams_global");
+ expect(res.headers["X-RateLimit-Limit"]).toBe("4");
+ expect(res.headers["X-GFS-RateLimit-Scope"]).toBe("active_part_streams_global");
expect(res.json).toMatchObject({
ok: false,
error: { code: "quota_exceeded", retryAfterSeconds: 1, limit: "active_part_streams_global" },
diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.ts
index 179933b77..f5ada831d 100644
--- a/gfs-controller/src/api/serve.ts
+++ b/gfs-controller/src/api/serve.ts
@@ -296,7 +296,7 @@ export class GfsServingHandler {
this.deps.metrics?.recordWrite(this.clock() - started);
}
} catch (err) {
- const { status, body } = toResponse(err);
+ const { status, body, rateLimitLimit } = toResponse(err);
if (status >= 500) {
// A 5xx must never be invisible: an unknown throw (e.g. the base64
// RangeError that produced a bogus `internal` 500) previously left no
@@ -313,7 +313,10 @@ export class GfsServingHandler {
if (!res.headersSent) {
const retryAfter = body.error.retryAfterSeconds;
if (retryAfter !== undefined) res.setHeader("Retry-After", String(retryAfter));
- if (body.error.limit !== undefined) res.setHeader("X-RateLimit-Limit", body.error.limit);
+ if (rateLimitLimit !== undefined)
+ res.setHeader("X-RateLimit-Limit", String(rateLimitLimit));
+ if (body.error.limit !== undefined)
+ res.setHeader("X-GFS-RateLimit-Scope", body.error.limit);
sendJson(res, status, body, abortConnectionHeader(req));
}
else res.end(); // a content stream already started; just terminate
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index 228eeea02..4ebdfa5e9 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -60,10 +60,22 @@ class MemoryDb {
}
if (
sql.startsWith(
- "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()"
+ "SELECT upload_id FROM gfs_upload_sessions WHERE expires_at <= now() AND ( state IN ('initiated','uploading','paused') OR (state = 'finalizing' AND finalizing_started_at IS NULL) )"
)
) {
- return { rows: [] }
+ return {
+ rows: this.sessions
+ .filter(row => {
+ const expired = Date.parse(String(row.expires_at)) <= Date.now()
+ const state = String(row.state)
+ return (
+ expired &&
+ (['initiated', 'uploading', 'paused'].includes(state) ||
+ (state === 'finalizing' && row.finalizing_started_at == null))
+ )
+ })
+ .map(row => ({ upload_id: row.upload_id })),
+ }
}
if (sql.startsWith('SELECT p.upload_id, p.part_number, p.offset_bytes, p.length_bytes')) {
return { rows: [] }
@@ -350,6 +362,13 @@ class MemoryDb {
row.session_epoch = Number(row.session_epoch) + 1
return { rows: [] }
}
+ if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'expired'")) {
+ const row = this.sessions.find(session => session.upload_id === String(values[0]))!
+ row.state = 'expired'
+ row.session_epoch = Number(row.session_epoch) + 1
+ row.active_part_count = 0
+ return { rows: [] }
+ }
if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {
const row = this.sessions.find(session => session.upload_id === String(values[0]))!
row.state = 'finalizing'
@@ -757,6 +776,27 @@ describe('GfsUploadSessionService', () => {
expect(Date.parse(db.sessions[0]!.expires_at as string)).toBeGreaterThan(originalExpiry)
})
+ it('expires a finalizing session whose durable start timestamp is missing', async () => {
+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-null-start-'))
+ const db = new MemoryDb()
+ const uploads = service(db)
+ const created = await uploads.create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'finalizing-null-start.bin',
+ sizeBytes: 0,
+ idempotencyKey: '6a6a6a6a-6a6a-4a6a-8a6a-6a6a6a6a6a6a',
+ })
+ const row = db.sessions.find(session => session.upload_id === created.session.uploadId)!
+ row.state = 'finalizing'
+ row.finalizing_started_at = null
+ row.expires_at = new Date(Date.now() - 1_000).toISOString()
+
+ await expect(uploads.reconcile()).resolves.toMatchObject({ expiredSessions: 1 })
+ expect(row).toMatchObject({ state: 'expired', active_part_count: 0 })
+ })
+
it('preserves a published final path until reconciliation resolves an ambiguous completion commit', async () => {
tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-final-path-reconcile-'))
const db = new MemoryDb()
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index a9d750ec3..16763e625 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -266,10 +266,11 @@ function requireEnabled(config: GfsUploadConfig): void {
if (!config.enabled) throw new GfsError('forbidden', 'resumable GFS uploads are disabled')
}
-function quotaExceeded(message: string, limit: string): GfsError {
+function quotaExceeded(message: string, limit: string, limitValue: number): GfsError {
return new GfsError('quota_exceeded', message, undefined, {
retryAfterSeconds: 1,
limit,
+ rateLimitLimit: limitValue,
})
}
@@ -592,7 +593,8 @@ export class GfsUploadSessionService {
if (Number(subjectActive.rows[0]?.count ?? 0) >= this.deps.config.maxActivePerSubject) {
throw quotaExceeded(
'active upload session subject quota reached',
- 'active_sessions_subject'
+ 'active_sessions_subject',
+ this.deps.config.maxActivePerSubject
)
}
const globalActive = await client.query(
@@ -603,7 +605,8 @@ export class GfsUploadSessionService {
if (Number(globalActive.rows[0]?.count ?? 0) >= this.deps.config.maxActiveGlobal) {
throw quotaExceeded(
'active upload session global quota reached',
- 'active_sessions_global'
+ 'active_sessions_global',
+ this.deps.config.maxActiveGlobal
)
}
if (available !== null) {
@@ -840,7 +843,8 @@ export class GfsUploadSessionService {
) {
throw quotaExceeded(
'global upload stream concurrency limit reached',
- 'active_part_streams_global'
+ 'active_part_streams_global',
+ this.deps.config.maxConcurrentPartStreamsGlobal
)
}
// The scalar on the session is a receipt/cache field. Admission must
@@ -856,7 +860,8 @@ export class GfsUploadSessionService {
if (activeSessionCount >= this.deps.config.maxConcurrentPartsPerSession)
throw quotaExceeded(
'per-session part concurrency limit reached',
- 'active_part_streams_session'
+ 'active_part_streams_session',
+ this.deps.config.maxConcurrentPartsPerSession
)
const stagingPath = sessionPath(
this.deps.storageMountPath,
@@ -1361,13 +1366,22 @@ export class GfsUploadSessionService {
}
const expired = await client.query(
`SELECT upload_id FROM gfs_upload_sessions
- WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()
+ WHERE expires_at <= now()
+ AND (
+ state IN ('initiated','uploading','paused')
+ OR (state = 'finalizing' AND finalizing_started_at IS NULL)
+ )
LIMIT 100
FOR UPDATE SKIP LOCKED`,
[]
)
for (const row of expired.rows) {
const uploadId = String(row.upload_id)
+ // A local finalizer may belong to an older rolling-deploy process that
+ // did not persist its start timestamp. Let that process settle before
+ // expiring the row; the next reconciliation fences it if it remains
+ // expired, while avoiding a local self-cancel during the same cycle.
+ if (this.inFlightFinalizers.has(uploadId)) continue
uploadDirs.set(uploadId, { purgeReceipt: false })
this.canceledUploads.add(uploadId)
await client.query(
@@ -1570,7 +1584,8 @@ export class GfsUploadSessionService {
if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {
throw quotaExceeded(
'upload finalization concurrency limit reached',
- 'active_finalizations'
+ 'active_finalizations',
+ this.deps.config.maxConcurrentFinalizations
)
}
if (!locked.wholeSha256 && expectedWhole) {