Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): recover expired finalizing uploads safely
Alfredo Lopez2a6d0b26b1e4
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex d6369407a..bfa0d0a98 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.451",+ "version": "0.1.452","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.451",+ "version": "0.1.452","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex a9613931e..1459f79eb 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.451",+ "version": "0.1.452","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.tsindex 33f2fea08..23a688c6c 100644--- a/control-api/src/middleware/gfsUploadAdmission.ts+++ b/control-api/src/middleware/gfsUploadAdmission.ts@@ -112,6 +112,7 @@ function sendBackendUnavailable(res: Response): void {function sendRateLimited(res: Response, limit: string, retryAfter: number, max: number): void {res.setHeader('Retry-After', String(retryAfter))res.setHeader('X-RateLimit-Limit', String(max))+ res.setHeader('X-GFS-RateLimit-Scope', limit)res.setHeader('X-RateLimit-Remaining', '0')res.status(429).json({error: 'gfs_upload_rate_limited',diff --git a/control-api/src/routes/gfs/proxy.ts b/control-api/src/routes/gfs/proxy.tsindex 4ce938444..36a335589 100644--- a/control-api/src/routes/gfs/proxy.ts+++ b/control-api/src/routes/gfs/proxy.ts@@ -30,6 +30,7 @@ const PASSTHROUGH_RESPONSE_HEADERS = ['retry-after','x-ratelimit-limit','x-ratelimit-remaining',+ 'x-gfs-ratelimit-scope',]const UUID_PATH = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'const UPLOAD_PART_PATH = new RegExp(`^/v1/uploads/${UUID_PATH}/parts/[0-9]+(?:\\?|$)`, 'i')diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.tsindex c38970a36..d073cc6ee 100644--- a/control-api/test/gfsUploadAdmission.test.ts+++ b/control-api/test/gfsUploadAdmission.test.ts@@ -163,6 +163,8 @@ describe('GFS upload admission', () => {expect(response.status).toBe(429)expect(response.headers['retry-after']).toMatch(/^\d+$/)+ expect(response.headers['x-ratelimit-limit']).toBe('120')+ expect(response.headers['x-gfs-ratelimit-scope']).toBe('principal_requests')expect(response.body).toMatchObject({error: 'gfs_upload_rate_limited',limit: 'principal_requests',diff --git a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts b/control-api/test/gfsUploadSession.realPostgres.integration.test.tsindex 6b4783180..5af683795 100644--- a/control-api/test/gfsUploadSession.realPostgres.integration.test.ts+++ b/control-api/test/gfsUploadSession.realPostgres.integration.test.ts@@ -836,6 +836,40 @@ describeRealPostgres('GFS Upload v2 session engine on real PostgreSQL', () => {expect(new Date(String(row.rows[0]?.expires_at)).getTime()).toBeGreaterThan(Date.now())})+ it('expires a finalizing session with a missing durable start timestamp after its TTL', async () => {+ const created = await uploads.create({+ drive,+ ownerSubject: owner,+ primarySubject: owner,+ operation: 'create',+ parentRid: 'b'.repeat(32),+ name: 'finalizing-null-start.bin',+ sizeBytes: 0,+ idempotencyKey: randomUUID(),+ })+ await pool.query(+ `UPDATE gfs_upload_sessions+ SET state = 'finalizing',+ finalizing_started_at = NULL,+ expires_at = now() - interval '1 second'+ WHERE upload_id = $1`,+ [created.session.uploadId]+ )++ const result = await uploads.reconcile()+ expect(result.expiredSessions).toBeGreaterThanOrEqual(1)+ const row = await pool.query(+ `SELECT state, finalizing_started_at, active_part_count+ FROM gfs_upload_sessions WHERE upload_id = $1`,+ [created.session.uploadId]+ )+ expect(row.rows[0]).toEqual({+ state: 'expired',+ finalizing_started_at: null,+ active_part_count: 0,+ })+ })+it('rejects cancel during finalization and leaves one completed receipt', async () => {let finalizerStarted!: () => voidlet releaseFinalizer!: () => voiddiff --git a/control-api/test/routes.gfsProxy.test.ts b/control-api/test/routes.gfsProxy.test.tsindex 33abb5edd..7e8cded9f 100644--- a/control-api/test/routes.gfsProxy.test.ts+++ b/control-api/test/routes.gfsProxy.test.ts@@ -208,7 +208,8 @@ describe('/api/v1/gfs/proxy', () => {headers: {'content-type': 'application/json','retry-after': '3',- 'x-ratelimit-limit': 'active_part_streams_global',+ 'x-ratelimit-limit': '4',+ 'x-gfs-ratelimit-scope': 'active_part_streams_global','x-ratelimit-remaining': '0','upload-length': '209715200',},@@ -221,7 +222,8 @@ describe('/api/v1/gfs/proxy', () => {expect(res.status).toBe(429)expect(res.headers['retry-after']).toBe('3')- expect(res.headers['x-ratelimit-limit']).toBe('active_part_streams_global')+ expect(res.headers['x-ratelimit-limit']).toBe('4')+ expect(res.headers['x-gfs-ratelimit-scope']).toBe('active_part_streams_global')expect(res.headers['x-ratelimit-remaining']).toBe('0')expect(res.headers['upload-length']).toBe('209715200')})diff --git a/control-ui/components/GfsBrowser.tsx b/control-ui/components/GfsBrowser.tsxindex e984c7280..c55ff8db8 100644--- a/control-ui/components/GfsBrowser.tsx+++ b/control-ui/components/GfsBrowser.tsx@@ -449,17 +449,7 @@ export function GfsBrowser(): React.JSX.Element {assertGfsFileUploadSize(file.size)const name = await normalizeGfsResourceName(file.name)const target = { operation: 'create' as const, parentRid: rid }- const pending = readPendingGfsUpload()- resumeUploadId =- pending &&- pending.fileName === file.name &&- pending.fileSize === file.size &&- pending.lastModified === file.lastModified &&- pending.target.operation === target.operation &&- pending.target.parentRid === target.parentRid &&- pending.name === name- ? pending.uploadId- : undefined+ resumeUploadId = matchingPendingResumeUploadId(file, target, name)const job = createGfsUploadJob({file,name,diff --git a/control-ui/components/__tests__/GfsBrowser.test.tsx b/control-ui/components/__tests__/GfsBrowser.test.tsxindex 2a2d74e1c..9718293b2 100644--- a/control-ui/components/__tests__/GfsBrowser.test.tsx+++ b/control-ui/components/__tests__/GfsBrowser.test.tsx@@ -316,6 +316,46 @@ describe('GfsBrowser', () => {)})+ it('reuses a matching persisted session from the button upload path', async () => {+ const rootId = '11111111-1111-1111-1111-111111111111'+ const rootRid = '11111111111111111111111111111111'+ const uploadId = '88888888-8888-4888-8888-888888888888'+ const lastModified = 1_725_000_000_000+ const fileName = 'button-resume.md'+ window.localStorage.setItem(+ 'evenfire:gfs-upload-v2:pending',+ JSON.stringify({+ uploadId,+ fileName,+ fileSize: 11,+ lastModified,+ target: { operation: 'create', parentRid: rootRid },+ name: fileName,+ })+ )+ mockApiGet.mockResolvedValue({ rootResourceId: rootId, items: [], nextCursor: null })+ renderBrowser()+ await screen.findByText('No resources are visible in this folder.')++ fireEvent.click(screen.getByRole('button', { name: /upload file/i }))+ const dialog = await screen.findByRole('dialog', { name: 'Upload file' })+ fireEvent.change(within(dialog).getByLabelText('Choose file to upload'), {+ target: { files: [new File(['button data'], fileName, { lastModified })] },+ })+ fireEvent.click(within(dialog).getByRole('button', { name: 'Upload' }))++ await waitFor(() =>+ expect(mockCreateGfsUploadJob).toHaveBeenCalledWith(+ expect.objectContaining({+ file: expect.any(File),+ name: fileName,+ resumeUploadId: uploadId,+ target: { operation: 'create', parentRid: rootRid },+ })+ )+ )+ })+it('uploads dropped images and Markdown files into the current folder', async () => {const rootId = '11111111-1111-1111-1111-111111111111'const rootRid = '11111111111111111111111111111111'diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 58ebe4a5b..af98647a2 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.427",+ "version": "0.1.428","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.427",+ "version": "0.1.428","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/llm-providers": "file:../packages/llm-providers",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 1d8336375..83f040783 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.427",+ "version": "0.1.428","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex 8e512a40f..660d39ec3 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.76",+ "version": "0.1.77","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.76",+ "version": "0.1.77","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex 604223d6d..714f06ede 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.76",+ "version": "0.1.77","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.tsindex b971ac89a..ba6348093 100644--- a/external-rest-api/src/releaseManifest.ts+++ b/external-rest-api/src/releaseManifest.ts@@ -8,7 +8,7 @@ export type ReleaseManifest = {export const releaseManifest: ReleaseManifest = {releaseId: 'v0.7.0',- externalRestApiVersion: '0.1.76',+ externalRestApiVersion: '0.1.77',rpcProxyVersion: '0.1.64',desktopVersion: '0.7.0',minimumDesktopVersion: '0.1.252',diff --git a/external-rest-api/src/routes/gfs.ts b/external-rest-api/src/routes/gfs.tsindex 04967eebb..7c1bea9fc 100644--- a/external-rest-api/src/routes/gfs.ts+++ b/external-rest-api/src/routes/gfs.ts@@ -48,6 +48,7 @@ const STREAM_HEADERS = ['retry-after','x-ratelimit-limit','x-ratelimit-remaining',+ 'x-gfs-ratelimit-scope',]function forwardControlApiError(error: unknown, res: Response, next: NextFunction): void {@@ -97,6 +98,7 @@ export function createGfsRouter(options: GfsRouterOptions = {}): Router {const retryAfterSeconds = Math.max(1, Math.ceil((resetAt - Date.now()) / 1000))res.setHeader('Retry-After', String(retryAfterSeconds))res.setHeader('X-RateLimit-Limit', String(info?.limit ?? edgeRequestLimit))+ res.setHeader('X-GFS-RateLimit-Scope', 'public_edge_requests')res.setHeader('X-RateLimit-Remaining', '0')res.status(429).json({error: 'gfs_upload_rate_limited',diff --git a/external-rest-api/test/routes.gfs.test.ts b/external-rest-api/test/routes.gfs.test.tsindex 0973698b8..e105bc32d 100644--- a/external-rest-api/test/routes.gfs.test.ts+++ b/external-rest-api/test/routes.gfs.test.ts@@ -70,6 +70,7 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',expect(second.status).toBe(429)expect(second.headers['retry-after']).toMatch(/^[1-9][0-9]*$/)expect(second.headers['x-ratelimit-limit']).toBe('1')+ expect(second.headers['x-gfs-ratelimit-scope']).toBe('public_edge_requests')expect(second.headers['x-ratelimit-remaining']).toBe('0')expect(second.body).toEqual({error: 'gfs_upload_rate_limited',@@ -261,7 +262,11 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)','rate limited',429,{ error: 'gfs_upload_rate_limited', limit: 'principal_bytes', retryAfterSeconds: 7 },- new Headers({ 'retry-after': '7' })+ new Headers({+ 'retry-after': '7',+ 'x-ratelimit-limit': '512',+ 'x-gfs-ratelimit-scope': 'principal_bytes',+ })))const response = await request(buildApp())@@ -269,6 +274,8 @@ describe('routes/gfs /me/gfs/* (user session passthrough → /external/gfs/*)',.set('authorization', 'Bearer sess-xyz')expect(response.status).toBe(429)expect(response.headers['retry-after']).toBe('7')+ expect(response.headers['x-ratelimit-limit']).toBe('512')+ expect(response.headers['x-gfs-ratelimit-scope']).toBe('principal_bytes')expect(response.body).toEqual({error: 'gfs_upload_rate_limited',limit: 'principal_bytes',diff --git a/gfs-controller/src/api/envelope.ts b/gfs-controller/src/api/envelope.tsindex 8980e6ac7..54281852e 100644--- a/gfs-controller/src/api/envelope.ts+++ b/gfs-controller/src/api/envelope.ts@@ -43,9 +43,13 @@ export function fail(error: GfsError): ErrEnvelope {* declared code/status; anything else is an `internal` (500) — failures are* surfaced, never swallowed into a misleading success.*/-export function toResponse(error: unknown): { status: number; body: ErrEnvelope } {+export function toResponse(error: unknown): {+ status: number;+ body: ErrEnvelope;+ rateLimitLimit?: number;+} {if (error instanceof GfsError) {- return { status: error.status, body: fail(error) };+ return { status: error.status, body: fail(error), rateLimitLimit: error.rateLimitLimit };}// Some lower layers throw typed errors carrying a `.code` that matches the// envelope vocabulary (auth, blob store). Honor it when present; otherwisediff --git a/gfs-controller/src/api/errors.ts b/gfs-controller/src/api/errors.tsindex 498bdff27..3ba87c480 100644--- a/gfs-controller/src/api/errors.ts+++ b/gfs-controller/src/api/errors.ts@@ -68,12 +68,13 @@ export class GfsError extends Error {readonly reason?: string;readonly retryAfterSeconds?: number;readonly limit?: string;+ readonly rateLimitLimit?: number;constructor(code: GfsErrorCode,message: string,reason?: string,- details?: { retryAfterSeconds?: number; limit?: string },+ details?: { retryAfterSeconds?: number; limit?: string; rateLimitLimit?: number },) {super(message);this.name = "GfsError";@@ -81,6 +82,7 @@ export class GfsError extends Error {this.reason = reason;this.retryAfterSeconds = details?.retryAfterSeconds;this.limit = details?.limit;+ this.rateLimitLimit = details?.rateLimitLimit;}get status(): number {diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.tsindex 61d010187..70cdb7199 100644--- a/gfs-controller/src/api/serve.test.ts+++ b/gfs-controller/src/api/serve.test.ts@@ -279,7 +279,7 @@ describe("GFS Upload v2 route contract", () => {"quota_exceeded","global upload stream concurrency limit reached",undefined,- { retryAfterSeconds: 1, limit: "active_part_streams_global" },+ { retryAfterSeconds: 1, limit: "active_part_streams_global", rateLimitLimit: 4 },);await run(uploadDeps({@@ -295,7 +295,8 @@ describe("GFS Upload v2 route contract", () => {);expect(res.statusCode).toBe(429);expect(res.headers["Retry-After"]).toBe("1");- expect(res.headers["X-RateLimit-Limit"]).toBe("active_part_streams_global");+ expect(res.headers["X-RateLimit-Limit"]).toBe("4");+ expect(res.headers["X-GFS-RateLimit-Scope"]).toBe("active_part_streams_global");expect(res.json).toMatchObject({ok: false,error: { code: "quota_exceeded", retryAfterSeconds: 1, limit: "active_part_streams_global" },diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.tsindex 179933b77..f5ada831d 100644--- a/gfs-controller/src/api/serve.ts+++ b/gfs-controller/src/api/serve.ts@@ -296,7 +296,7 @@ export class GfsServingHandler {this.deps.metrics?.recordWrite(this.clock() - started);}} catch (err) {- const { status, body } = toResponse(err);+ const { status, body, rateLimitLimit } = toResponse(err);if (status >= 500) {// A 5xx must never be invisible: an unknown throw (e.g. the base64// RangeError that produced a bogus `internal` 500) previously left no@@ -313,7 +313,10 @@ export class GfsServingHandler {if (!res.headersSent) {const retryAfter = body.error.retryAfterSeconds;if (retryAfter !== undefined) res.setHeader("Retry-After", String(retryAfter));- if (body.error.limit !== undefined) res.setHeader("X-RateLimit-Limit", body.error.limit);+ if (rateLimitLimit !== undefined)+ res.setHeader("X-RateLimit-Limit", String(rateLimitLimit));+ if (body.error.limit !== undefined)+ res.setHeader("X-GFS-RateLimit-Scope", body.error.limit);sendJson(res, status, body, abortConnectionHeader(req));}else res.end(); // a content stream already started; just terminatediff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex 228eeea02..4ebdfa5e9 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -60,10 +60,22 @@ class MemoryDb {}if (sql.startsWith(- "SELECT upload_id FROM gfs_upload_sessions WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()"+ "SELECT upload_id FROM gfs_upload_sessions WHERE expires_at <= now() AND ( state IN ('initiated','uploading','paused') OR (state = 'finalizing' AND finalizing_started_at IS NULL) )")) {- return { rows: [] }+ return {+ rows: this.sessions+ .filter(row => {+ const expired = Date.parse(String(row.expires_at)) <= Date.now()+ const state = String(row.state)+ return (+ expired &&+ (['initiated', 'uploading', 'paused'].includes(state) ||+ (state === 'finalizing' && row.finalizing_started_at == null))+ )+ })+ .map(row => ({ upload_id: row.upload_id })),+ }}if (sql.startsWith('SELECT p.upload_id, p.part_number, p.offset_bytes, p.length_bytes')) {return { rows: [] }@@ -350,6 +362,13 @@ class MemoryDb {row.session_epoch = Number(row.session_epoch) + 1return { rows: [] }}+ if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'expired'")) {+ const row = this.sessions.find(session => session.upload_id === String(values[0]))!+ row.state = 'expired'+ row.session_epoch = Number(row.session_epoch) + 1+ row.active_part_count = 0+ return { rows: [] }+ }if (sql.startsWith("UPDATE gfs_upload_sessions SET state = 'finalizing'")) {const row = this.sessions.find(session => session.upload_id === String(values[0]))!row.state = 'finalizing'@@ -757,6 +776,27 @@ describe('GfsUploadSessionService', () => {expect(Date.parse(db.sessions[0]!.expires_at as string)).toBeGreaterThan(originalExpiry)})+ it('expires a finalizing session whose durable start timestamp is missing', async () => {+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-finalizing-null-start-'))+ const db = new MemoryDb()+ const uploads = service(db)+ const created = await uploads.create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'finalizing-null-start.bin',+ sizeBytes: 0,+ idempotencyKey: '6a6a6a6a-6a6a-4a6a-8a6a-6a6a6a6a6a6a',+ })+ const row = db.sessions.find(session => session.upload_id === created.session.uploadId)!+ row.state = 'finalizing'+ row.finalizing_started_at = null+ row.expires_at = new Date(Date.now() - 1_000).toISOString()++ await expect(uploads.reconcile()).resolves.toMatchObject({ expiredSessions: 1 })+ expect(row).toMatchObject({ state: 'expired', active_part_count: 0 })+ })+it('preserves a published final path until reconciliation resolves an ambiguous completion commit', async () => {tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-final-path-reconcile-'))const db = new MemoryDb()diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex a9d750ec3..16763e625 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -266,10 +266,11 @@ function requireEnabled(config: GfsUploadConfig): void {if (!config.enabled) throw new GfsError('forbidden', 'resumable GFS uploads are disabled')}-function quotaExceeded(message: string, limit: string): GfsError {+function quotaExceeded(message: string, limit: string, limitValue: number): GfsError {return new GfsError('quota_exceeded', message, undefined, {retryAfterSeconds: 1,limit,+ rateLimitLimit: limitValue,})}@@ -592,7 +593,8 @@ export class GfsUploadSessionService {if (Number(subjectActive.rows[0]?.count ?? 0) >= this.deps.config.maxActivePerSubject) {throw quotaExceeded('active upload session subject quota reached',- 'active_sessions_subject'+ 'active_sessions_subject',+ this.deps.config.maxActivePerSubject)}const globalActive = await client.query(@@ -603,7 +605,8 @@ export class GfsUploadSessionService {if (Number(globalActive.rows[0]?.count ?? 0) >= this.deps.config.maxActiveGlobal) {throw quotaExceeded('active upload session global quota reached',- 'active_sessions_global'+ 'active_sessions_global',+ this.deps.config.maxActiveGlobal)}if (available !== null) {@@ -840,7 +843,8 @@ export class GfsUploadSessionService {) {throw quotaExceeded('global upload stream concurrency limit reached',- 'active_part_streams_global'+ 'active_part_streams_global',+ this.deps.config.maxConcurrentPartStreamsGlobal)}// The scalar on the session is a receipt/cache field. Admission must@@ -856,7 +860,8 @@ export class GfsUploadSessionService {if (activeSessionCount >= this.deps.config.maxConcurrentPartsPerSession)throw quotaExceeded('per-session part concurrency limit reached',- 'active_part_streams_session'+ 'active_part_streams_session',+ this.deps.config.maxConcurrentPartsPerSession)const stagingPath = sessionPath(this.deps.storageMountPath,@@ -1361,13 +1366,22 @@ export class GfsUploadSessionService {}const expired = await client.query(`SELECT upload_id FROM gfs_upload_sessions- WHERE state IN ('initiated','uploading','paused') AND expires_at <= now()+ WHERE expires_at <= now()+ AND (+ state IN ('initiated','uploading','paused')+ OR (state = 'finalizing' AND finalizing_started_at IS NULL)+ )LIMIT 100FOR UPDATE SKIP LOCKED`,[])for (const row of expired.rows) {const uploadId = String(row.upload_id)+ // A local finalizer may belong to an older rolling-deploy process that+ // did not persist its start timestamp. Let that process settle before+ // expiring the row; the next reconciliation fences it if it remains+ // expired, while avoiding a local self-cancel during the same cycle.+ if (this.inFlightFinalizers.has(uploadId)) continueuploadDirs.set(uploadId, { purgeReceipt: false })this.canceledUploads.add(uploadId)await client.query(@@ -1570,7 +1584,8 @@ export class GfsUploadSessionService {if (Number(finalizing.rows[0]?.count ?? 0) >= this.deps.config.maxConcurrentFinalizations) {throw quotaExceeded('upload finalization concurrency limit reached',- 'active_finalizations'+ 'active_finalizations',+ this.deps.config.maxConcurrentFinalizations)}if (!locked.wholeSha256 && expectedWhole) {