Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs): close Zach round-three upload gaps
Alfredo Lopezac361bd38ea6
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.tsindex 530f87552..33f2fea08 100644--- a/control-api/src/middleware/gfsUploadAdmission.ts+++ b/control-api/src/middleware/gfsUploadAdmission.ts@@ -122,9 +122,12 @@ function sendRateLimited(res: Response, limit: string, retryAfter: number, max:/*** Replica-safe upload-v2 admission. Fixed-window request/weighted-byte- * counters and active-request slots all live in PostgreSQL; the incoming part+ * counters and active stream slots all live in PostgreSQL; the incoming part* stream is not consumed here. `proxyUploadToGfsc` separately counts observed* bytes while forwarding so a false Content-Length cannot bypass this charge.+ * The active slot is intentionally acquired only for a part PUT: lifecycle+ * reads/actions must remain available to pause, reconcile, or cancel a stream+ * that is already consuming a slot.*/export function gfsUploadAdmission(req: GfsUploadAdmissionRequest,@@ -209,6 +212,12 @@ export function gfsUploadAdmission(}}+ if (!isPart) {+ gfsUploadAdmissionRequestsTotal.inc({ limit: 'all', result: 'allowed' }, 1)+ next()+ return+ }+const lease = await acquireRateLimitConcurrencyLease([{ bucketKey: 'gfs-upload:active:global', maxConcurrent: config.gfsUploadMaxActiveGlobal },{diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.tsindex f7e44cf3d..948ffc203 100644--- a/control-api/src/services/rateLimiterService.ts+++ b/control-api/src/services/rateLimiterService.ts@@ -183,6 +183,10 @@ type AdvisoryLockKey = {function advisoryKey(value: string): AdvisoryLockKey {const digest = createHash('sha256').update(value).digest()+ // PostgreSQL's two-int advisory-lock form is one 64-bit lock identity. Keep+ // both digest halves; only the bounded slot number is folded into the low+ // half, so the collision domain remains the full 64-bit key (not a single+ // 32-bit hash).return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }}@@ -287,8 +291,8 @@ export async function acquireRateLimitConcurrencyLease(for (let slot = 0; slot < requirement.maxConcurrent; slot += 1) {// Keep both halves of the SHA-256-derived key in the PostgreSQL// advisory lock identity. The slot is folded into the low half so- // distinct buckets do not collide merely because their first 32- // digest bits match.+ // distinct slots remain independent without reducing the bucket to+ // a 32-bit advisory key.const key = { high: bucketKey.high, low: bucketKey.low ^ slot }const identity = `${key.high}:${key.low}`if (heldSlotsFor(client).has(identity)) continuediff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.tsindex 93cd1c50c..c38970a36 100644--- a/control-api/test/gfsUploadAdmission.test.ts+++ b/control-api/test/gfsUploadAdmission.test.ts@@ -260,20 +260,17 @@ describe('GFS upload admission', () => {expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')})- it('returns the same 429 contract when the active-request budget is exhausted', async () => {+ it('does not consume an active stream slot for lifecycle requests', async () => {acquireMock.mockResolvedValueOnce({allowed: false,backendAvailable: true,release: releaseMock,})const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')- expect(response.status).toBe(429)- expect(response.headers['retry-after']).toBe('1')- expect(response.body).toEqual({- error: 'gfs_upload_rate_limited',- limit: 'active_requests',- retryAfterSeconds: 1,- })+ expect(response.status).toBe(200)+ expect(response.body).toEqual({ ok: true })+ expect(acquireMock).not.toHaveBeenCalled()+ expect(metricActiveInc).not.toHaveBeenCalled()})it('fails closed when PostgreSQL admission is unavailable', async () => {diff --git a/control-ui/app/control-api/[...path]/route.ts b/control-ui/app/control-api/[...path]/route.tsindex e8e6dba9d..52b647a5a 100644--- a/control-ui/app/control-api/[...path]/route.ts+++ b/control-ui/app/control-api/[...path]/route.ts@@ -45,6 +45,7 @@ function resolveGfsUploadProxyTimeoutMs(): number {// default. Set CONTROL_UI_PROXY_MAX_BODY_BYTES on the deployment to change it.const DEFAULT_MAX_BODY_BYTES = 24 * 1024 * 1024const MAX_BODY_BYTES_CEILING = 512 * 1024 * 1024+const GFS_UPLOAD_MAX_PART_BYTES = 16 * 1024 * 1024function resolveMaxBodyBytes(): number {const raw = process.env.CONTROL_UI_PROXY_MAX_BODY_BYTESconst parsed = raw ? Number(raw) : Number.NaN@@ -186,17 +187,27 @@ async function proxyControlApi(const hasBody = req.method !== 'GET' && req.method !== 'HEAD'let body: ArrayBuffer | undefined- let streamingBody: ReadableStream<Uint8Array> | null = nullif (rawUpload) {const declaredChunk = Number(req.headers.get('upload-chunk-length'))if (!Number.isSafeInteger(declaredChunk) ||declaredChunk <= 0 ||- declaredChunk > 16 * 1024 * 1024+ declaredChunk > GFS_UPLOAD_MAX_PART_BYTES) {return NextResponse.json({ error: 'payload_too_large' }, { status: 413 })}- streamingBody = req.body+ // Count the actual stream before forwarding it. The header is an admission+ // hint, not a security boundary: without this bounded read a caller could+ // declare a small part and stream an unbounded body through the UI proxy.+ const read = await readBodyCapped(req.body, GFS_UPLOAD_MAX_PART_BYTES)+ if (read === 'too-large') {+ return NextResponse.json({ error: 'payload_too_large' }, { status: 413 })+ }+ const observedBytes = read?.byteLength ?? 0+ if (observedBytes !== declaredChunk) {+ return NextResponse.json({ error: 'upload_length_mismatch' }, { status: 400 })+ }+ body = read ?? undefined} else if (hasBody) {const cap = resolveMaxBodyBytes()// Fast path: reject a declared-oversize length before reading a single byte.@@ -226,12 +237,11 @@ async function proxyControlApi(const fetchInit: RequestInit & { duplex?: 'half' } = {method: req.method,headers,- body: rawUpload ? streamingBody : body,+ body,cache: 'no-store',redirect: 'manual',signal,}- if (rawUpload) fetchInit.duplex = 'half'const upstream = await fetch(upstreamUrl, fetchInit)return new NextResponse(upstream.body, {diff --git a/control-ui/lib/__tests__/controlApiProxyRoute.test.ts b/control-ui/lib/__tests__/controlApiProxyRoute.test.tsindex 465090d27..592a1ad4b 100644--- a/control-ui/lib/__tests__/controlApiProxyRoute.test.ts+++ b/control-ui/lib/__tests__/controlApiProxyRoute.test.ts@@ -295,6 +295,103 @@ describe('control-ui control-api proxy route', () => {})})+ describe('raw GFS part bodies are counted before forwarding', () => {+ const path = [+ 'api',+ 'v1',+ 'gfs',+ 'proxy',+ 'v1',+ 'uploads',+ 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',+ 'parts',+ '0',+ ]++ it('rejects a streamed part that exceeds the declared length', async () => {+ const fetchMock = vi+ .spyOn(globalThis, 'fetch')+ .mockResolvedValue(new Response(null, { status: 204 }))+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array(3))+ controller.enqueue(new Uint8Array(2))+ controller.close()+ },+ })+ const req = new NextRequest(+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',+ {+ method: 'PUT',+ headers: { 'Upload-Chunk-Length': '4' },+ body: stream,+ duplex: 'half',+ } as RequestInit & { duplex: 'half' }+ )++ const res = await POST(req, { params: { path } })++ expect(res.status).toBe(400)+ await expect(res.json()).resolves.toEqual({ error: 'upload_length_mismatch' })+ expect(fetchMock).not.toHaveBeenCalled()+ })++ it('rejects a streamed part above the hard 16 MiB cap even when the header lies', async () => {+ const fetchMock = vi+ .spyOn(globalThis, 'fetch')+ .mockResolvedValue(new Response(null, { status: 204 }))+ const stream = new ReadableStream<Uint8Array>({+ start(controller) {+ controller.enqueue(new Uint8Array(16 * 1024 * 1024))+ controller.enqueue(new Uint8Array(1))+ controller.close()+ },+ })+ const req = new NextRequest(+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',+ {+ method: 'PUT',+ headers: { 'Upload-Chunk-Length': String(16 * 1024 * 1024) },+ body: stream,+ duplex: 'half',+ } as RequestInit & { duplex: 'half' }+ )++ const res = await POST(req, { params: { path } })++ expect(res.status).toBe(413)+ await expect(res.json()).resolves.toEqual({ error: 'payload_too_large' })+ expect(fetchMock).not.toHaveBeenCalled()+ })++ it('forwards the exact counted bytes as a bounded ArrayBuffer', async () => {+ const fetchMock = vi+ .spyOn(globalThis, 'fetch')+ .mockResolvedValue(new Response(null, { status: 204 }))+ const payload = new Uint8Array([1, 2, 3, 4])+ const req = new NextRequest(+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',+ {+ method: 'PUT',+ headers: { 'Upload-Chunk-Length': String(payload.byteLength) },+ body: new ReadableStream({+ start(controller) {+ controller.enqueue(payload)+ controller.close()+ },+ }),+ duplex: 'half',+ } as RequestInit & { duplex: 'half' }+ )++ const res = await POST(req, { params: { path } })+ const [, init] = fetchMock.mock.calls[0] as [string, RequestInit]++ expect(res.status).toBe(204)+ expect(Buffer.from(init.body as ArrayBuffer)).toEqual(Buffer.from(payload))+ })+ })+describe('header handling', () => {afterEach(() => vi.restoreAllMocks())diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.tsindex 579816668..e6b7a7a10 100644--- a/desktop-app/src/__tests__/appService.gfs.test.ts+++ b/desktop-app/src/__tests__/appService.gfs.test.ts@@ -325,6 +325,30 @@ describe('AppService GFS upload security scope', () => {}})+ it('never falls back to a second legacy resource for an explicit resume', async () => {+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-explicit-resume-no-fallback-'))+ try {+ const filePath = join(root, 'resume.bin')+ await writeFile(filePath, Buffer.from('resume payload'))+ const service = authenticatedUploadService(join(root, 'gfs-upload-sessions.json'))+ service.startDesktopGfsUpload.mockRejectedValue(+ new DesktopUploadCapabilityError('resumable uploads are disabled')+ )+ service.gfsClient = {+ createResource: vi.fn(),+ replaceFile: vi.fn(),+ }++ await expect(+ service.startGfsFileUpload('parent-rid', 'resume.bin', filePath, 'main', 'resume-upload-id')+ ).rejects.toBeInstanceOf(DesktopUploadCapabilityError)+ expect(service.gfsClient.createResource).not.toHaveBeenCalled()+ expect(service.gfsClient.replaceFile).not.toHaveBeenCalled()+ } finally {+ await rm(root, { recursive: true, force: true })+ }+ })+it('keeps existing upload controls and read state available during a transient team hop', async () => {const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-transient-controls-'))try {@@ -368,16 +392,11 @@ describe('AppService GFS upload security scope', () => {try {const statePath = join(root, 'gfs-upload-sessions.json')const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')+ await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))const service = authenticatedUploadService(statePath)- // The normal state loader converts orphaned active records to- // suspended_auth. This injects the already-migrated persisted branch so- // the IPC boundary itself is still proven not to leak `active`.- ;(service as any).readDesktopGfsUploadState = vi- .fn()- .mockResolvedValue({ version: 2, records: [record], quarantined: [] })await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({- state: 'uploading',+ state: 'suspended_auth',uploadedBytes: 0,totalBytes: 4,})diff --git a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsxindex 20b486a26..b6e2e816f 100644--- a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx+++ b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx@@ -257,6 +257,65 @@ describe('useGfsBrowserController', () => {expect(getUploadSnapshot).not.toHaveBeenCalled()})+ it('rehydrates a scoped persisted upload after the renderer mounts', async () => {+ const listUploadSessions = vi.fn(async () => [+ {+ uploadId: 'terminal-upload',+ fileName: 'done.bin',+ fileSize: 12,+ name: 'done.bin',+ drive: 'main',+ status: 'failed' as const,+ target: { operation: 'create' as const, parentRid: 'parent-rid' },+ },+ {+ uploadId: 'persisted-upload',+ fileName: 'resume.bin',+ fileSize: 12,+ name: 'resume.bin',+ drive: 'main',+ status: 'suspended_auth' as const,+ target: { operation: 'create' as const, parentRid: 'parent-rid' },+ },+ ])+ const getUploadSnapshot = vi.fn(async () => ({+ state: 'suspended_auth' as const,+ session: {+ uploadId: 'persisted-upload',+ state: 'paused',+ expectedBytes: 12,+ committedBytes: 8,+ },+ uploadedBytes: 8,+ totalBytes: 12,+ }))+ Object.defineProperty(window, 'clerum', {+ configurable: true,+ value: {+ gfs: {+ listUploadSessions,+ getUploadSnapshot,+ listAccessible: vi.fn(async () => ({ items: [], nextCursor: null })),+ listChildren: vi.fn(async () => ({ items: [], nextCursor: null })),+ affordances: vi.fn(async () => ({+ held: [],+ canDelegate: false,+ grantableBits: [],+ canCreateShare: false,+ })),+ },+ },+ })++ render(<UploadProbe />, { wrapper: Harness })++ await waitFor(() =>+ expect(screen.getByTestId('upload-state').textContent).toBe('suspended_auth')+ )+ expect(listUploadSessions).toHaveBeenCalledWith('main')+ expect(getUploadSnapshot).toHaveBeenCalledWith('persisted-upload', 'main')+ })+it('refreshes cached affordances after permissions change outside Desktop', async () => {let held = ['read']const affordances = vi.fn(async () => ({diff --git a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.tsindex 3fb242271..aa37ccd11 100644--- a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts+++ b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts@@ -111,6 +111,7 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {const [uploadSnapshot, setUploadSnapshot] = useState<GfsUploadSnapshot | null>(null)const uploadIdRef = useRef<string | null>(null)const previousSessionScopeRef = useRef<string | null>(null)+ const uploadRehydrateGenerationRef = useRef(0)const current = crumbs.length ? crumbs[crumbs.length - 1] : nullconst currentIsDirectory = current?.kind === 'directory'@@ -285,6 +286,47 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {}}, [])+ // Persisted Desktop sessions are the source of truth after an app restart.+ // Rehydrate one scoped session on mount/scope change so FilesPage can render+ // the same progress and pause/resume/cancel controls it renders for a live+ // upload. The IPC list is already owner/team/environment/drive scoped; the+ // second snapshot read supplies the durable byte counters and state.+ useEffect(() => {+ const generation = uploadRehydrateGenerationRef.current + 1+ uploadRehydrateGenerationRef.current = generation+ if (!sessionScope) {+ uploadIdRef.current = null+ setUploadSnapshot(null)+ return+ }+ const listUploadSessions = window.clerum?.gfs?.listUploadSessions+ if (typeof listUploadSessions !== 'function') return+ let disposed = false+ void (async () => {+ try {+ const sessions = await listUploadSessions(DRIVE)+ if (disposed || uploadRehydrateGenerationRef.current !== generation) return+ const persisted = sessions.find(+ session =>+ session.drive === DRIVE &&+ (session.status === 'active' ||+ session.status === 'paused' ||+ session.status === 'suspended_auth')+ )+ if (!persisted) return+ uploadIdRef.current = persisted.uploadId+ await waitForUpload(persisted.uploadId)+ } catch (error) {+ if (!disposed && uploadRehydrateGenerationRef.current === generation) {+ setOpenError(toMessage(error))+ }+ }+ })()+ return () => {+ disposed = true+ }+ }, [sessionScope, waitForUpload])+const startFileUpload = useCallback(async (input: {parentResourceId: stringdiff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.jsonindex e7eecf31b..9348ad2a8 100644--- a/external-rest-api/package-lock.json+++ b/external-rest-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-external-rest-api",- "version": "0.1.70",+ "version": "0.1.71","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-external-rest-api",- "version": "0.1.70",+ "version": "0.1.71","dependencies": {"cors": "^2.8.5","express": "^4.21.2",diff --git a/external-rest-api/package.json b/external-rest-api/package.jsonindex fe0cdaf72..f9ca2d12f 100644--- a/external-rest-api/package.json+++ b/external-rest-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-external-rest-api",- "version": "0.1.70",+ "version": "0.1.71","description": "External REST API for profile, team management, and RPC token brokerage","main": "dist/main.js","engines": {diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.tsindex 903f66fa4..48e652c45 100644--- a/external-rest-api/src/app.ts+++ b/external-rest-api/src/app.ts@@ -26,6 +26,12 @@ const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/paexport function createApp() {const app = express()+ // The deployment admits traffic only from the cloudflared ingress namespace+ // (see deploy/base/profiles/networkpolicies.yaml). That single trusted hop+ // overwrites the client address before this process, so req.ip is the stable+ // limiter identity. Direct service exposure would invalidate this contract+ // and must be rejected by deployment policy rather than by accepting a+ // caller-controlled X-Forwarded-For chain here.app.set('trust proxy', 1)app.use(cors({diff --git a/external-rest-api/src/middleware/rateLimit.ts b/external-rest-api/src/middleware/rateLimit.tsindex 2a8635dbf..ac731c030 100644--- a/external-rest-api/src/middleware/rateLimit.ts+++ b/external-rest-api/src/middleware/rateLimit.ts@@ -25,6 +25,10 @@ export function createRateLimiter(options: {}, windowMs).unref()return function rateLimitMiddleware(req: Request, res: Response, next: NextFunction): void {+ // Each route owns its limiter instance, so auth/me/invitation budgets do+ // not share counters with GFS. The key is intentionally evaluated after+ // Express trust-proxy processing; callers must not be allowed to choose a+ // different X-Forwarded-For chain by reaching this service directly.const key = keyFn(req)const now = Date.now()let bucket = buckets.get(key)diff --git a/external-rest-api/test/routes.accessForwarding.test.ts b/external-rest-api/test/routes.accessForwarding.test.tsindex daf541e51..c371cfcdf 100644--- a/external-rest-api/test/routes.accessForwarding.test.ts+++ b/external-rest-api/test/routes.accessForwarding.test.ts@@ -165,11 +165,14 @@ describe('routes/access forwarding', () => {.expect(200)}- await request(app)+ const limited = await request(app).get('/me/teams/directory').set('authorization', 'Bearer good-token').expect(429)+ expect(limited.headers['retry-after']).toMatch(/^\d+$/)+ expect(limited.headers['x-ratelimit-limit']).toBe('10')+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')expect(meServiceMock.getMyTeamDirectory).toHaveBeenCalledTimes(10)})diff --git a/external-rest-api/test/routes.auth.test.ts b/external-rest-api/test/routes.auth.test.tsindex e3f10ef4c..04b1849d1 100644--- a/external-rest-api/test/routes.auth.test.ts+++ b/external-rest-api/test/routes.auth.test.ts@@ -7,6 +7,7 @@ import { createAuthRouter } from '../src/routes/auth.js'const authServiceMock = vi.hoisted(() => ({loginWithGoogle: vi.fn(),loginWithPassword: vi.fn(),+ requestPasswordReset: vi.fn(),}))vi.mock('../src/services/authService.js', () => authServiceMock)@@ -29,6 +30,7 @@ function buildApp() {describe('routes/auth password-login', () => {beforeEach(() => {authServiceMock.loginWithPassword.mockReset()+ authServiceMock.requestPasswordReset.mockReset()})it('propagates invalid credentials as a 401 instead of a 500', async () => {@@ -97,4 +99,30 @@ describe('routes/auth password-login', () => {expect(res.body.me.email).toBe('[email protected]')expect(String(res.headers['set-cookie'])).toContain('profile_session=desktop-session-jwt')})++ it('documents the password-reset limiter headers and client-IP contract', async () => {+ authServiceMock.requestPasswordReset.mockResolvedValue({ requested: true })+ const app = buildApp()+ const email = '[email protected]'++ for (let attempt = 0; attempt < 5; attempt += 1) {+ const res = await request(app)+ .post('/api/v1/auth/password-reset/request')+ .set('x-forwarded-for', '198.51.100.44')+ .send({ email })+ .expect(200)+ expect(res.headers['x-ratelimit-limit']).toBe('5')+ expect(res.headers['x-ratelimit-remaining']).toBe(String(4 - attempt))+ }++ const limited = await request(app)+ .post('/api/v1/auth/password-reset/request')+ .set('x-forwarded-for', '198.51.100.44')+ .send({ email })+ .expect(429)+ expect(limited.headers['retry-after']).toMatch(/^\d+$/)+ expect(limited.headers['x-ratelimit-limit']).toBe('5')+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')+ expect(limited.body).toMatchObject({ retryAfterSeconds: expect.any(Number) })+ })})diff --git a/external-rest-api/test/routes.invitations.test.ts b/external-rest-api/test/routes.invitations.test.tsindex a08a2f62f..b112bc927 100644--- a/external-rest-api/test/routes.invitations.test.ts+++ b/external-rest-api/test/routes.invitations.test.ts@@ -199,10 +199,14 @@ describe('routes/invitations', () => {.expect(400)}- await request(app)+ const limited = await request(app).post('/invitations/password').set('x-forwarded-for', '198.51.100.10').send({}).expect(429)++ expect(limited.headers['retry-after']).toMatch(/^\d+$/)+ expect(limited.headers['x-ratelimit-limit']).toBe('10')+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')})})diff --git a/gfs-controller/src/cleanup.test.ts b/gfs-controller/src/cleanup.test.tsnew file mode 100644index 000000000..7475f5e23--- /dev/null+++ b/gfs-controller/src/cleanup.test.ts@@ -0,0 +1,39 @@+import { describe, expect, it, vi } from 'vitest'+import { createWriterCleanupRunner } from './cleanup'++describe('writer cleanup runner', () => {+ it('runs upload reconciliation only after the service is available at invocation time', async () => {+ let uploadService: { reconcile: () => Promise<void> } | undefined+ const reconcileBlobs = vi.fn(async () => undefined)+ const run = createWriterCleanupRunner({+ reconcileUploads: async () => uploadService?.reconcile(),+ reconcileBlobs,+ })++ uploadService = { reconcile: vi.fn(async () => undefined) }+ await run()++ expect(uploadService.reconcile).toHaveBeenCalledTimes(1)+ expect(reconcileBlobs).toHaveBeenCalledTimes(1)+ })++ it('does not overlap a running pass and releases the gate after failures', async () => {+ let release!: () => void+ const blocked = new Promise<void>(resolve => {+ release = resolve+ })+ const reconcileUploads = vi.fn(async () => blocked)+ const reconcileBlobs = vi.fn(async () => undefined)+ const run = createWriterCleanupRunner({ reconcileUploads, reconcileBlobs })++ const first = run()+ const second = run()+ expect(reconcileUploads).toHaveBeenCalledTimes(1)+ release()+ await Promise.all([first, second])+ await run()++ expect(reconcileUploads).toHaveBeenCalledTimes(2)+ expect(reconcileBlobs).toHaveBeenCalledTimes(2)+ })+})diff --git a/gfs-controller/src/cleanup.ts b/gfs-controller/src/cleanup.tsnew file mode 100644index 000000000..8cb44621f--- /dev/null+++ b/gfs-controller/src/cleanup.ts@@ -0,0 +1,45 @@+export interface WriterCleanupRunnerDeps {+ reconcileUploads?: () => Promise<void>+ reconcileBlobs: () => Promise<void>+ onBlobFailure?: () => void+ log?: (message: string) => void+}++/**+ * Serialize the writer's two durable cleanup passes.+ *+ * The runner is deliberately created before the serving handler but invoked+ * only after all writer services have been wired. This keeps boot cleanup and+ * the interval on one gate without allowing a synchronous first invocation to+ * observe an uninitialized upload service.+ */+export function createWriterCleanupRunner(deps: WriterCleanupRunnerDeps): () => Promise<void> {+ let running = false+ const log = deps.log ?? ((message: string) => console.error(message))++ return async () => {+ if (running) return+ running = true+ try {+ if (deps.reconcileUploads) {+ try {+ await deps.reconcileUploads()+ } catch (error) {+ log(+ `[gfsc] upload reconciliation failed: ${error instanceof Error ? error.message : String(error)}`+ )+ }+ }+ try {+ await deps.reconcileBlobs()+ } catch (error) {+ deps.onBlobFailure?.()+ log(+ `[gfsc] blob reconciliation failed: ${error instanceof Error ? error.message : String(error)}`+ )+ }+ } finally {+ running = false+ }+ }+}diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.tsindex af5ae9a0e..5e2e80e81 100644--- a/gfs-controller/src/index.ts+++ b/gfs-controller/src/index.ts@@ -12,6 +12,7 @@ import { InvalidationListener, ListenClient } from "./authz/invalidation";import { DbAuditSink, PermissionClient } from "./authz/permissionClient";import { createPermissionStoreProbe } from "./authz/storeProbe";import { resolveAuthzContext } from "./authz/subjectResolver";+import { createWriterCleanupRunner } from "./cleanup";import { loadConfig } from "./config";import { PgResourceStore } from "./db/resourceStore";import { GfsWriteService, PgTransactor } from "./db/writeStore";@@ -178,35 +179,21 @@ async function main(): Promise<void> {: undefined;if (config.storageRole === "writer") {- let cleanupRunning = false;- const cleanup = async (): Promise<void> => {- if (cleanupRunning) return;- cleanupRunning = true;- try {- if (uploadServiceForCleanup) {- try {- await uploadServiceForCleanup.reconcile();- } catch (err) {- console.error(`[gfsc] upload reconciliation failed: ${errMsg(err)}`);- }- }- try {- await reconcileExpiredBlobs(blobManifests!, blobs, metrics, {- olderThanMs: config.blobCleanupSafetyWindowMs,- limit: config.blobCleanupBatchSize,- });- } catch (err) {- metrics.recordBlobCleanupFailure();- console.error(`[gfsc] blob reconciliation failed: ${errMsg(err)}`);- }- } finally {- cleanupRunning = false;- }- };- runCleanup = cleanup;- cleanupTimer = setInterval(() => void cleanup(), config.blobCleanupIntervalMs);+ runCleanup = createWriterCleanupRunner({+ reconcileUploads: async () => {+ await uploadServiceForCleanup?.reconcile();+ },+ reconcileBlobs: async () => {+ await reconcileExpiredBlobs(blobManifests!, blobs, metrics, {+ olderThanMs: config.blobCleanupSafetyWindowMs,+ limit: config.blobCleanupBatchSize,+ });+ },+ onBlobFailure: () => metrics.recordBlobCleanupFailure(),+ log: (message) => console.error(message),+ });+ cleanupTimer = setInterval(() => void runCleanup?.(), config.blobCleanupIntervalMs);cleanupTimer.unref();- void cleanup();}serving = new GfsServingHandler({diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.tsindex 6b0476048..6ad8c90f9 100644--- a/gfs-controller/src/upload/uploadSession.test.ts+++ b/gfs-controller/src/upload/uploadSession.test.ts@@ -207,6 +207,15 @@ class MemoryDb {)return { rows }}+ if (sql.startsWith('SELECT state, lease_epoch FROM gfs_upload_parts')) {+ const [uploadId, partNumber] = values+ return {+ rows: this.parts.filter(+ part =>+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)+ ),+ }+ }if (sql.startsWith('INSERT INTO gfs_upload_parts')) {const [uploadId, partNumber, offset, length, sha, staging, epoch] = valuesconst row: Row = {@@ -409,6 +418,22 @@ class ThrowAfterCommitTransactor implements Transactor {}}+class ThrowBeforePartCommitTransactor implements Transactor {+ private transactionCount = 0++ constructor(private readonly delegate: Transactor) {}++ async transaction<T>(+ fn: (client: TxClient) => Promise<T>,+ options?: TransactionOptions+ ): Promise<T> {+ this.transactionCount += 1+ if (this.transactionCount === 2)+ throw new CommitOutcomeUnknownError(new Error('synthetic pre-commit failure'))+ return this.delegate.transaction(fn, options)+ }+}+async function settleWithin<T>(promise: Promise<T>,timeoutMs = 500@@ -544,6 +569,43 @@ describe('GfsUploadSessionService', () => {expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')})+ it('removes a rename-before-commit orphan while the reserved lease is still locked', async () => {+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-orphan-'))+ const db = new MemoryDb()+ const tx = new ThrowBeforePartCommitTransactor({ transaction: async fn => fn(db) })+ const created = await service(db).create({+ ...principal,+ operation: 'create',+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',+ name: 'orphan.bin',+ sizeBytes: TEST_CONFIG.preferredPartBytes,+ idempotencyKey: '67676767-6767-4676-8676-676767676767',+ })+ const uploads = service(db, TEST_CONFIG, { tx })+ const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x41)+ const finalPath = join(tempRoot, '.uploads', created.session.uploadId, 'parts', '0.part')++ await expect(+ uploads.putPart(+ created.session.uploadId,+ partGeometry(+ {+ expectedBytes: TEST_CONFIG.preferredPartBytes,+ partBytes: TEST_CONFIG.preferredPartBytes,+ },+ 0+ ),+ createSha(payload),+ Readable.from([payload]) as never,+ principal+ )+ ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)++ await expect(stat(finalPath)).rejects.toMatchObject({ code: 'ENOENT' })+ expect(db.parts[0]).toMatchObject({ state: 'failed' })+ expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })+ })+it('cancels terminally and rejects a conflicting idempotency fingerprint', async () => {tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))const db = new MemoryDb()diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.tsindex c0d8e73f8..89d9ddd7b 100644--- a/gfs-controller/src/upload/uploadSession.ts+++ b/gfs-controller/src/upload/uploadSession.ts@@ -1013,14 +1013,33 @@ export class GfsUploadSessionService {if (reconciliation)return { part: reconciliation.part, session: toReceipt(reconciliation.session) }await removeIfPresent(stagingPath)- if (finalPath && reconciliation !== undefined) await removeIfPresent(finalPath)await this.deps.tx.transaction(async client => {+ // A definitive row lock separates a pre-commit failure from an+ // outcome-unknown commit. Only a still-reserved row with the same+ // lease may safely lose its computed final path; the lock fences a+ // fresh reservation from racing that removal.+ const currentPart = await client.query(+ `SELECT state, lease_epoch FROM gfs_upload_parts+ WHERE upload_id = $1 AND part_number = $2 FOR UPDATE`,+ [uploadId, part.partNumber]+ )+ const current = currentPart.rows[0] as+ | { state?: unknown; lease_epoch?: unknown }+ | undefined+ if (+ !current ||+ current.state !== 'reserved' ||+ Number(current.lease_epoch) !== reserved.leaseEpoch+ )+ return+ if (finalPath && !(await removeStrict(finalPath)))+ throw new Error('could not remove an orphaned committed-part path')const failedPart = await client.query(`UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL- WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'- RETURNING 1`,- [uploadId, part.partNumber]+ WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved' AND lease_epoch = $3+ RETURNING 1`,+ [uploadId, part.partNumber, reserved.leaseEpoch])// The commit transaction already decremented the durable counter when// it reached PostgreSQL. Only a part that was still reserved needs a