Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs): close Zach round-three upload gaps

Alfredo Lopezac361bd38ea6
diff --git a/control-api/src/middleware/gfsUploadAdmission.ts b/control-api/src/middleware/gfsUploadAdmission.ts
index 530f87552..33f2fea08 100644
--- a/control-api/src/middleware/gfsUploadAdmission.ts
+++ b/control-api/src/middleware/gfsUploadAdmission.ts
@@ -122,9 +122,12 @@ function sendRateLimited(res: Response, limit: string, retryAfter: number, max:
/**
* Replica-safe upload-v2 admission. Fixed-window request/weighted-byte
- * counters and active-request slots all live in PostgreSQL; the incoming part
+ * counters and active stream slots all live in PostgreSQL; the incoming part
* stream is not consumed here. `proxyUploadToGfsc` separately counts observed
* bytes while forwarding so a false Content-Length cannot bypass this charge.
+ * The active slot is intentionally acquired only for a part PUT: lifecycle
+ * reads/actions must remain available to pause, reconcile, or cancel a stream
+ * that is already consuming a slot.
*/
export function gfsUploadAdmission(
req: GfsUploadAdmissionRequest,
@@ -209,6 +212,12 @@ export function gfsUploadAdmission(
}
}
+ if (!isPart) {
+ gfsUploadAdmissionRequestsTotal.inc({ limit: 'all', result: 'allowed' }, 1)
+ next()
+ return
+ }
+
const lease = await acquireRateLimitConcurrencyLease([
{ bucketKey: 'gfs-upload:active:global', maxConcurrent: config.gfsUploadMaxActiveGlobal },
{
diff --git a/control-api/src/services/rateLimiterService.ts b/control-api/src/services/rateLimiterService.ts
index f7e44cf3d..948ffc203 100644
--- a/control-api/src/services/rateLimiterService.ts
+++ b/control-api/src/services/rateLimiterService.ts
@@ -183,6 +183,10 @@ type AdvisoryLockKey = {
function advisoryKey(value: string): AdvisoryLockKey {
const digest = createHash('sha256').update(value).digest()
+ // PostgreSQL's two-int advisory-lock form is one 64-bit lock identity. Keep
+ // both digest halves; only the bounded slot number is folded into the low
+ // half, so the collision domain remains the full 64-bit key (not a single
+ // 32-bit hash).
return { high: digest.readInt32BE(0), low: digest.readInt32BE(4) }
}
@@ -287,8 +291,8 @@ export async function acquireRateLimitConcurrencyLease(
for (let slot = 0; slot < requirement.maxConcurrent; slot += 1) {
// Keep both halves of the SHA-256-derived key in the PostgreSQL
// advisory lock identity. The slot is folded into the low half so
- // distinct buckets do not collide merely because their first 32
- // digest bits match.
+ // distinct slots remain independent without reducing the bucket to
+ // a 32-bit advisory key.
const key = { high: bucketKey.high, low: bucketKey.low ^ slot }
const identity = `${key.high}:${key.low}`
if (heldSlotsFor(client).has(identity)) continue
diff --git a/control-api/test/gfsUploadAdmission.test.ts b/control-api/test/gfsUploadAdmission.test.ts
index 93cd1c50c..c38970a36 100644
--- a/control-api/test/gfsUploadAdmission.test.ts
+++ b/control-api/test/gfsUploadAdmission.test.ts
@@ -260,20 +260,17 @@ describe('GFS upload admission', () => {
expect(checkMock.mock.calls[1][0]).toBe('gfs-upload:req:ip:203.0.113.7')
})
- it('returns the same 429 contract when the active-request budget is exhausted', async () => {
+ it('does not consume an active stream slot for lifecycle requests', async () => {
acquireMock.mockResolvedValueOnce({
allowed: false,
backendAvailable: true,
release: releaseMock,
})
const response = await request(buildLifecycleApp()).get('/external/gfs/capabilities')
- expect(response.status).toBe(429)
- expect(response.headers['retry-after']).toBe('1')
- expect(response.body).toEqual({
- error: 'gfs_upload_rate_limited',
- limit: 'active_requests',
- retryAfterSeconds: 1,
- })
+ expect(response.status).toBe(200)
+ expect(response.body).toEqual({ ok: true })
+ expect(acquireMock).not.toHaveBeenCalled()
+ expect(metricActiveInc).not.toHaveBeenCalled()
})
it('fails closed when PostgreSQL admission is unavailable', async () => {
diff --git a/control-ui/app/control-api/[...path]/route.ts b/control-ui/app/control-api/[...path]/route.ts
index e8e6dba9d..52b647a5a 100644
--- a/control-ui/app/control-api/[...path]/route.ts
+++ b/control-ui/app/control-api/[...path]/route.ts
@@ -45,6 +45,7 @@ function resolveGfsUploadProxyTimeoutMs(): number {
// default. Set CONTROL_UI_PROXY_MAX_BODY_BYTES on the deployment to change it.
const DEFAULT_MAX_BODY_BYTES = 24 * 1024 * 1024
const MAX_BODY_BYTES_CEILING = 512 * 1024 * 1024
+const GFS_UPLOAD_MAX_PART_BYTES = 16 * 1024 * 1024
function resolveMaxBodyBytes(): number {
const raw = process.env.CONTROL_UI_PROXY_MAX_BODY_BYTES
const parsed = raw ? Number(raw) : Number.NaN
@@ -186,17 +187,27 @@ async function proxyControlApi(
const hasBody = req.method !== 'GET' && req.method !== 'HEAD'
let body: ArrayBuffer | undefined
- let streamingBody: ReadableStream<Uint8Array> | null = null
if (rawUpload) {
const declaredChunk = Number(req.headers.get('upload-chunk-length'))
if (
!Number.isSafeInteger(declaredChunk) ||
declaredChunk <= 0 ||
- declaredChunk > 16 * 1024 * 1024
+ declaredChunk > GFS_UPLOAD_MAX_PART_BYTES
) {
return NextResponse.json({ error: 'payload_too_large' }, { status: 413 })
}
- streamingBody = req.body
+ // Count the actual stream before forwarding it. The header is an admission
+ // hint, not a security boundary: without this bounded read a caller could
+ // declare a small part and stream an unbounded body through the UI proxy.
+ const read = await readBodyCapped(req.body, GFS_UPLOAD_MAX_PART_BYTES)
+ if (read === 'too-large') {
+ return NextResponse.json({ error: 'payload_too_large' }, { status: 413 })
+ }
+ const observedBytes = read?.byteLength ?? 0
+ if (observedBytes !== declaredChunk) {
+ return NextResponse.json({ error: 'upload_length_mismatch' }, { status: 400 })
+ }
+ body = read ?? undefined
} else if (hasBody) {
const cap = resolveMaxBodyBytes()
// Fast path: reject a declared-oversize length before reading a single byte.
@@ -226,12 +237,11 @@ async function proxyControlApi(
const fetchInit: RequestInit & { duplex?: 'half' } = {
method: req.method,
headers,
- body: rawUpload ? streamingBody : body,
+ body,
cache: 'no-store',
redirect: 'manual',
signal,
}
- if (rawUpload) fetchInit.duplex = 'half'
const upstream = await fetch(upstreamUrl, fetchInit)
return new NextResponse(upstream.body, {
diff --git a/control-ui/lib/__tests__/controlApiProxyRoute.test.ts b/control-ui/lib/__tests__/controlApiProxyRoute.test.ts
index 465090d27..592a1ad4b 100644
--- a/control-ui/lib/__tests__/controlApiProxyRoute.test.ts
+++ b/control-ui/lib/__tests__/controlApiProxyRoute.test.ts
@@ -295,6 +295,103 @@ describe('control-ui control-api proxy route', () => {
})
})
+ describe('raw GFS part bodies are counted before forwarding', () => {
+ const path = [
+ 'api',
+ 'v1',
+ 'gfs',
+ 'proxy',
+ 'v1',
+ 'uploads',
+ 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+ 'parts',
+ '0',
+ ]
+
+ it('rejects a streamed part that exceeds the declared length', async () => {
+ const fetchMock = vi
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValue(new Response(null, { status: 204 }))
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new Uint8Array(3))
+ controller.enqueue(new Uint8Array(2))
+ controller.close()
+ },
+ })
+ const req = new NextRequest(
+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',
+ {
+ method: 'PUT',
+ headers: { 'Upload-Chunk-Length': '4' },
+ body: stream,
+ duplex: 'half',
+ } as RequestInit & { duplex: 'half' }
+ )
+
+ const res = await POST(req, { params: { path } })
+
+ expect(res.status).toBe(400)
+ await expect(res.json()).resolves.toEqual({ error: 'upload_length_mismatch' })
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
+
+ it('rejects a streamed part above the hard 16 MiB cap even when the header lies', async () => {
+ const fetchMock = vi
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValue(new Response(null, { status: 204 }))
+ const stream = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(new Uint8Array(16 * 1024 * 1024))
+ controller.enqueue(new Uint8Array(1))
+ controller.close()
+ },
+ })
+ const req = new NextRequest(
+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',
+ {
+ method: 'PUT',
+ headers: { 'Upload-Chunk-Length': String(16 * 1024 * 1024) },
+ body: stream,
+ duplex: 'half',
+ } as RequestInit & { duplex: 'half' }
+ )
+
+ const res = await POST(req, { params: { path } })
+
+ expect(res.status).toBe(413)
+ await expect(res.json()).resolves.toEqual({ error: 'payload_too_large' })
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
+
+ it('forwards the exact counted bytes as a bounded ArrayBuffer', async () => {
+ const fetchMock = vi
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValue(new Response(null, { status: 204 }))
+ const payload = new Uint8Array([1, 2, 3, 4])
+ const req = new NextRequest(
+ 'http://localhost:3000/control-api/api/v1/gfs/proxy/v1/uploads/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa/parts/0',
+ {
+ method: 'PUT',
+ headers: { 'Upload-Chunk-Length': String(payload.byteLength) },
+ body: new ReadableStream({
+ start(controller) {
+ controller.enqueue(payload)
+ controller.close()
+ },
+ }),
+ duplex: 'half',
+ } as RequestInit & { duplex: 'half' }
+ )
+
+ const res = await POST(req, { params: { path } })
+ const [, init] = fetchMock.mock.calls[0] as [string, RequestInit]
+
+ expect(res.status).toBe(204)
+ expect(Buffer.from(init.body as ArrayBuffer)).toEqual(Buffer.from(payload))
+ })
+ })
+
describe('header handling', () => {
afterEach(() => vi.restoreAllMocks())
diff --git a/desktop-app/src/__tests__/appService.gfs.test.ts b/desktop-app/src/__tests__/appService.gfs.test.ts
index 579816668..e6b7a7a10 100644
--- a/desktop-app/src/__tests__/appService.gfs.test.ts
+++ b/desktop-app/src/__tests__/appService.gfs.test.ts
@@ -325,6 +325,30 @@ describe('AppService GFS upload security scope', () => {
}
})
+ it('never falls back to a second legacy resource for an explicit resume', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-explicit-resume-no-fallback-'))
+ try {
+ const filePath = join(root, 'resume.bin')
+ await writeFile(filePath, Buffer.from('resume payload'))
+ const service = authenticatedUploadService(join(root, 'gfs-upload-sessions.json'))
+ service.startDesktopGfsUpload.mockRejectedValue(
+ new DesktopUploadCapabilityError('resumable uploads are disabled')
+ )
+ service.gfsClient = {
+ createResource: vi.fn(),
+ replaceFile: vi.fn(),
+ }
+
+ await expect(
+ service.startGfsFileUpload('parent-rid', 'resume.bin', filePath, 'main', 'resume-upload-id')
+ ).rejects.toBeInstanceOf(DesktopUploadCapabilityError)
+ expect(service.gfsClient.createResource).not.toHaveBeenCalled()
+ expect(service.gfsClient.replaceFile).not.toHaveBeenCalled()
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('keeps existing upload controls and read state available during a transient team hop', async () => {
const root = await mkdtemp(join(tmpdir(), 'evenfire-gfs-transient-controls-'))
try {
@@ -368,16 +392,11 @@ describe('AppService GFS upload security scope', () => {
try {
const statePath = join(root, 'gfs-upload-sessions.json')
const record = scopedUploadRecord('93939393-9393-4393-8393-939393939393')
+ await writeFile(statePath, JSON.stringify({ version: 2, records: [record], quarantined: [] }))
const service = authenticatedUploadService(statePath)
- // The normal state loader converts orphaned active records to
- // suspended_auth. This injects the already-migrated persisted branch so
- // the IPC boundary itself is still proven not to leak `active`.
- ;(service as any).readDesktopGfsUploadState = vi
- .fn()
- .mockResolvedValue({ version: 2, records: [record], quarantined: [] })
await expect(service.getGfsUploadSnapshot(record.uploadId, 'main')).resolves.toMatchObject({
- state: 'uploading',
+ state: 'suspended_auth',
uploadedBytes: 0,
totalBytes: 4,
})
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
index 20b486a26..b6e2e816f 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/useGfsBrowserController.test.tsx
@@ -257,6 +257,65 @@ describe('useGfsBrowserController', () => {
expect(getUploadSnapshot).not.toHaveBeenCalled()
})
+ it('rehydrates a scoped persisted upload after the renderer mounts', async () => {
+ const listUploadSessions = vi.fn(async () => [
+ {
+ uploadId: 'terminal-upload',
+ fileName: 'done.bin',
+ fileSize: 12,
+ name: 'done.bin',
+ drive: 'main',
+ status: 'failed' as const,
+ target: { operation: 'create' as const, parentRid: 'parent-rid' },
+ },
+ {
+ uploadId: 'persisted-upload',
+ fileName: 'resume.bin',
+ fileSize: 12,
+ name: 'resume.bin',
+ drive: 'main',
+ status: 'suspended_auth' as const,
+ target: { operation: 'create' as const, parentRid: 'parent-rid' },
+ },
+ ])
+ const getUploadSnapshot = vi.fn(async () => ({
+ state: 'suspended_auth' as const,
+ session: {
+ uploadId: 'persisted-upload',
+ state: 'paused',
+ expectedBytes: 12,
+ committedBytes: 8,
+ },
+ uploadedBytes: 8,
+ totalBytes: 12,
+ }))
+ Object.defineProperty(window, 'clerum', {
+ configurable: true,
+ value: {
+ gfs: {
+ listUploadSessions,
+ getUploadSnapshot,
+ listAccessible: vi.fn(async () => ({ items: [], nextCursor: null })),
+ listChildren: vi.fn(async () => ({ items: [], nextCursor: null })),
+ affordances: vi.fn(async () => ({
+ held: [],
+ canDelegate: false,
+ grantableBits: [],
+ canCreateShare: false,
+ })),
+ },
+ },
+ })
+
+ render(<UploadProbe />, { wrapper: Harness })
+
+ await waitFor(() =>
+ expect(screen.getByTestId('upload-state').textContent).toBe('suspended_auth')
+ )
+ expect(listUploadSessions).toHaveBeenCalledWith('main')
+ expect(getUploadSnapshot).toHaveBeenCalledWith('persisted-upload', 'main')
+ })
+
it('refreshes cached affordances after permissions change outside Desktop', async () => {
let held = ['read']
const affordances = vi.fn(async () => ({
diff --git a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
index 3fb242271..aa37ccd11 100644
--- a/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
+++ b/desktop-app/ui/src/hooks/domain/useGfsBrowserController.ts
@@ -111,6 +111,7 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {
const [uploadSnapshot, setUploadSnapshot] = useState<GfsUploadSnapshot | null>(null)
const uploadIdRef = useRef<string | null>(null)
const previousSessionScopeRef = useRef<string | null>(null)
+ const uploadRehydrateGenerationRef = useRef(0)
const current = crumbs.length ? crumbs[crumbs.length - 1] : null
const currentIsDirectory = current?.kind === 'directory'
@@ -285,6 +286,47 @@ export function useGfsBrowserController(options: GfsBrowserControllerOptions = {
}
}, [])
+ // Persisted Desktop sessions are the source of truth after an app restart.
+ // Rehydrate one scoped session on mount/scope change so FilesPage can render
+ // the same progress and pause/resume/cancel controls it renders for a live
+ // upload. The IPC list is already owner/team/environment/drive scoped; the
+ // second snapshot read supplies the durable byte counters and state.
+ useEffect(() => {
+ const generation = uploadRehydrateGenerationRef.current + 1
+ uploadRehydrateGenerationRef.current = generation
+ if (!sessionScope) {
+ uploadIdRef.current = null
+ setUploadSnapshot(null)
+ return
+ }
+ const listUploadSessions = window.clerum?.gfs?.listUploadSessions
+ if (typeof listUploadSessions !== 'function') return
+ let disposed = false
+ void (async () => {
+ try {
+ const sessions = await listUploadSessions(DRIVE)
+ if (disposed || uploadRehydrateGenerationRef.current !== generation) return
+ const persisted = sessions.find(
+ session =>
+ session.drive === DRIVE &&
+ (session.status === 'active' ||
+ session.status === 'paused' ||
+ session.status === 'suspended_auth')
+ )
+ if (!persisted) return
+ uploadIdRef.current = persisted.uploadId
+ await waitForUpload(persisted.uploadId)
+ } catch (error) {
+ if (!disposed && uploadRehydrateGenerationRef.current === generation) {
+ setOpenError(toMessage(error))
+ }
+ }
+ })()
+ return () => {
+ disposed = true
+ }
+ }, [sessionScope, waitForUpload])
+
const startFileUpload = useCallback(
async (input: {
parentResourceId: string
diff --git a/external-rest-api/package-lock.json b/external-rest-api/package-lock.json
index e7eecf31b..9348ad2a8 100644
--- a/external-rest-api/package-lock.json
+++ b/external-rest-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.70",
+ "version": "0.1.71",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-external-rest-api",
- "version": "0.1.70",
+ "version": "0.1.71",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/external-rest-api/package.json b/external-rest-api/package.json
index fe0cdaf72..f9ca2d12f 100644
--- a/external-rest-api/package.json
+++ b/external-rest-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-external-rest-api",
- "version": "0.1.70",
+ "version": "0.1.71",
"description": "External REST API for profile, team management, and RPC token brokerage",
"main": "dist/main.js",
"engines": {
diff --git a/external-rest-api/src/app.ts b/external-rest-api/src/app.ts
index 903f66fa4..48e652c45 100644
--- a/external-rest-api/src/app.ts
+++ b/external-rest-api/src/app.ts
@@ -26,6 +26,12 @@ const GFS_UPLOAD_PART_PATH = new RegExp(`^/api/v1/me/gfs/uploads/${UUID_PATH}/pa
export function createApp() {
const app = express()
+ // The deployment admits traffic only from the cloudflared ingress namespace
+ // (see deploy/base/profiles/networkpolicies.yaml). That single trusted hop
+ // overwrites the client address before this process, so req.ip is the stable
+ // limiter identity. Direct service exposure would invalidate this contract
+ // and must be rejected by deployment policy rather than by accepting a
+ // caller-controlled X-Forwarded-For chain here.
app.set('trust proxy', 1)
app.use(
cors({
diff --git a/external-rest-api/src/middleware/rateLimit.ts b/external-rest-api/src/middleware/rateLimit.ts
index 2a8635dbf..ac731c030 100644
--- a/external-rest-api/src/middleware/rateLimit.ts
+++ b/external-rest-api/src/middleware/rateLimit.ts
@@ -25,6 +25,10 @@ export function createRateLimiter(options: {
}, windowMs).unref()
return function rateLimitMiddleware(req: Request, res: Response, next: NextFunction): void {
+ // Each route owns its limiter instance, so auth/me/invitation budgets do
+ // not share counters with GFS. The key is intentionally evaluated after
+ // Express trust-proxy processing; callers must not be allowed to choose a
+ // different X-Forwarded-For chain by reaching this service directly.
const key = keyFn(req)
const now = Date.now()
let bucket = buckets.get(key)
diff --git a/external-rest-api/test/routes.accessForwarding.test.ts b/external-rest-api/test/routes.accessForwarding.test.ts
index daf541e51..c371cfcdf 100644
--- a/external-rest-api/test/routes.accessForwarding.test.ts
+++ b/external-rest-api/test/routes.accessForwarding.test.ts
@@ -165,11 +165,14 @@ describe('routes/access forwarding', () => {
.expect(200)
}
- await request(app)
+ const limited = await request(app)
.get('/me/teams/directory')
.set('authorization', 'Bearer good-token')
.expect(429)
+ expect(limited.headers['retry-after']).toMatch(/^\d+$/)
+ expect(limited.headers['x-ratelimit-limit']).toBe('10')
+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')
expect(meServiceMock.getMyTeamDirectory).toHaveBeenCalledTimes(10)
})
diff --git a/external-rest-api/test/routes.auth.test.ts b/external-rest-api/test/routes.auth.test.ts
index e3f10ef4c..04b1849d1 100644
--- a/external-rest-api/test/routes.auth.test.ts
+++ b/external-rest-api/test/routes.auth.test.ts
@@ -7,6 +7,7 @@ import { createAuthRouter } from '../src/routes/auth.js'
const authServiceMock = vi.hoisted(() => ({
loginWithGoogle: vi.fn(),
loginWithPassword: vi.fn(),
+ requestPasswordReset: vi.fn(),
}))
vi.mock('../src/services/authService.js', () => authServiceMock)
@@ -29,6 +30,7 @@ function buildApp() {
describe('routes/auth password-login', () => {
beforeEach(() => {
authServiceMock.loginWithPassword.mockReset()
+ authServiceMock.requestPasswordReset.mockReset()
})
it('propagates invalid credentials as a 401 instead of a 500', async () => {
@@ -97,4 +99,30 @@ describe('routes/auth password-login', () => {
expect(res.body.me.email).toBe('[email protected]')
expect(String(res.headers['set-cookie'])).toContain('profile_session=desktop-session-jwt')
})
+
+ it('documents the password-reset limiter headers and client-IP contract', async () => {
+ authServiceMock.requestPasswordReset.mockResolvedValue({ requested: true })
+ const app = buildApp()
+ const email = '[email protected]'
+
+ for (let attempt = 0; attempt < 5; attempt += 1) {
+ const res = await request(app)
+ .post('/api/v1/auth/password-reset/request')
+ .set('x-forwarded-for', '198.51.100.44')
+ .send({ email })
+ .expect(200)
+ expect(res.headers['x-ratelimit-limit']).toBe('5')
+ expect(res.headers['x-ratelimit-remaining']).toBe(String(4 - attempt))
+ }
+
+ const limited = await request(app)
+ .post('/api/v1/auth/password-reset/request')
+ .set('x-forwarded-for', '198.51.100.44')
+ .send({ email })
+ .expect(429)
+ expect(limited.headers['retry-after']).toMatch(/^\d+$/)
+ expect(limited.headers['x-ratelimit-limit']).toBe('5')
+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')
+ expect(limited.body).toMatchObject({ retryAfterSeconds: expect.any(Number) })
+ })
})
diff --git a/external-rest-api/test/routes.invitations.test.ts b/external-rest-api/test/routes.invitations.test.ts
index a08a2f62f..b112bc927 100644
--- a/external-rest-api/test/routes.invitations.test.ts
+++ b/external-rest-api/test/routes.invitations.test.ts
@@ -199,10 +199,14 @@ describe('routes/invitations', () => {
.expect(400)
}
- await request(app)
+ const limited = await request(app)
.post('/invitations/password')
.set('x-forwarded-for', '198.51.100.10')
.send({})
.expect(429)
+
+ expect(limited.headers['retry-after']).toMatch(/^\d+$/)
+ expect(limited.headers['x-ratelimit-limit']).toBe('10')
+ expect(limited.headers['x-ratelimit-remaining']).toBe('0')
})
})
diff --git a/gfs-controller/src/cleanup.test.ts b/gfs-controller/src/cleanup.test.ts
new file mode 100644
index 000000000..7475f5e23
--- /dev/null
+++ b/gfs-controller/src/cleanup.test.ts
@@ -0,0 +1,39 @@
+import { describe, expect, it, vi } from 'vitest'
+import { createWriterCleanupRunner } from './cleanup'
+
+describe('writer cleanup runner', () => {
+ it('runs upload reconciliation only after the service is available at invocation time', async () => {
+ let uploadService: { reconcile: () => Promise<void> } | undefined
+ const reconcileBlobs = vi.fn(async () => undefined)
+ const run = createWriterCleanupRunner({
+ reconcileUploads: async () => uploadService?.reconcile(),
+ reconcileBlobs,
+ })
+
+ uploadService = { reconcile: vi.fn(async () => undefined) }
+ await run()
+
+ expect(uploadService.reconcile).toHaveBeenCalledTimes(1)
+ expect(reconcileBlobs).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not overlap a running pass and releases the gate after failures', async () => {
+ let release!: () => void
+ const blocked = new Promise<void>(resolve => {
+ release = resolve
+ })
+ const reconcileUploads = vi.fn(async () => blocked)
+ const reconcileBlobs = vi.fn(async () => undefined)
+ const run = createWriterCleanupRunner({ reconcileUploads, reconcileBlobs })
+
+ const first = run()
+ const second = run()
+ expect(reconcileUploads).toHaveBeenCalledTimes(1)
+ release()
+ await Promise.all([first, second])
+ await run()
+
+ expect(reconcileUploads).toHaveBeenCalledTimes(2)
+ expect(reconcileBlobs).toHaveBeenCalledTimes(2)
+ })
+})
diff --git a/gfs-controller/src/cleanup.ts b/gfs-controller/src/cleanup.ts
new file mode 100644
index 000000000..8cb44621f
--- /dev/null
+++ b/gfs-controller/src/cleanup.ts
@@ -0,0 +1,45 @@
+export interface WriterCleanupRunnerDeps {
+ reconcileUploads?: () => Promise<void>
+ reconcileBlobs: () => Promise<void>
+ onBlobFailure?: () => void
+ log?: (message: string) => void
+}
+
+/**
+ * Serialize the writer's two durable cleanup passes.
+ *
+ * The runner is deliberately created before the serving handler but invoked
+ * only after all writer services have been wired. This keeps boot cleanup and
+ * the interval on one gate without allowing a synchronous first invocation to
+ * observe an uninitialized upload service.
+ */
+export function createWriterCleanupRunner(deps: WriterCleanupRunnerDeps): () => Promise<void> {
+ let running = false
+ const log = deps.log ?? ((message: string) => console.error(message))
+
+ return async () => {
+ if (running) return
+ running = true
+ try {
+ if (deps.reconcileUploads) {
+ try {
+ await deps.reconcileUploads()
+ } catch (error) {
+ log(
+ `[gfsc] upload reconciliation failed: ${error instanceof Error ? error.message : String(error)}`
+ )
+ }
+ }
+ try {
+ await deps.reconcileBlobs()
+ } catch (error) {
+ deps.onBlobFailure?.()
+ log(
+ `[gfsc] blob reconciliation failed: ${error instanceof Error ? error.message : String(error)}`
+ )
+ }
+ } finally {
+ running = false
+ }
+ }
+}
diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.ts
index af5ae9a0e..5e2e80e81 100644
--- a/gfs-controller/src/index.ts
+++ b/gfs-controller/src/index.ts
@@ -12,6 +12,7 @@ import { InvalidationListener, ListenClient } from "./authz/invalidation";
import { DbAuditSink, PermissionClient } from "./authz/permissionClient";
import { createPermissionStoreProbe } from "./authz/storeProbe";
import { resolveAuthzContext } from "./authz/subjectResolver";
+import { createWriterCleanupRunner } from "./cleanup";
import { loadConfig } from "./config";
import { PgResourceStore } from "./db/resourceStore";
import { GfsWriteService, PgTransactor } from "./db/writeStore";
@@ -178,35 +179,21 @@ async function main(): Promise<void> {
: undefined;
if (config.storageRole === "writer") {
- let cleanupRunning = false;
- const cleanup = async (): Promise<void> => {
- if (cleanupRunning) return;
- cleanupRunning = true;
- try {
- if (uploadServiceForCleanup) {
- try {
- await uploadServiceForCleanup.reconcile();
- } catch (err) {
- console.error(`[gfsc] upload reconciliation failed: ${errMsg(err)}`);
- }
- }
- try {
- await reconcileExpiredBlobs(blobManifests!, blobs, metrics, {
- olderThanMs: config.blobCleanupSafetyWindowMs,
- limit: config.blobCleanupBatchSize,
- });
- } catch (err) {
- metrics.recordBlobCleanupFailure();
- console.error(`[gfsc] blob reconciliation failed: ${errMsg(err)}`);
- }
- } finally {
- cleanupRunning = false;
- }
- };
- runCleanup = cleanup;
- cleanupTimer = setInterval(() => void cleanup(), config.blobCleanupIntervalMs);
+ runCleanup = createWriterCleanupRunner({
+ reconcileUploads: async () => {
+ await uploadServiceForCleanup?.reconcile();
+ },
+ reconcileBlobs: async () => {
+ await reconcileExpiredBlobs(blobManifests!, blobs, metrics, {
+ olderThanMs: config.blobCleanupSafetyWindowMs,
+ limit: config.blobCleanupBatchSize,
+ });
+ },
+ onBlobFailure: () => metrics.recordBlobCleanupFailure(),
+ log: (message) => console.error(message),
+ });
+ cleanupTimer = setInterval(() => void runCleanup?.(), config.blobCleanupIntervalMs);
cleanupTimer.unref();
- void cleanup();
}
serving = new GfsServingHandler({
diff --git a/gfs-controller/src/upload/uploadSession.test.ts b/gfs-controller/src/upload/uploadSession.test.ts
index 6b0476048..6ad8c90f9 100644
--- a/gfs-controller/src/upload/uploadSession.test.ts
+++ b/gfs-controller/src/upload/uploadSession.test.ts
@@ -207,6 +207,15 @@ class MemoryDb {
)
return { rows }
}
+ if (sql.startsWith('SELECT state, lease_epoch FROM gfs_upload_parts')) {
+ const [uploadId, partNumber] = values
+ return {
+ rows: this.parts.filter(
+ part =>
+ part.upload_id === String(uploadId) && Number(part.part_number) === Number(partNumber)
+ ),
+ }
+ }
if (sql.startsWith('INSERT INTO gfs_upload_parts')) {
const [uploadId, partNumber, offset, length, sha, staging, epoch] = values
const row: Row = {
@@ -409,6 +418,22 @@ class ThrowAfterCommitTransactor implements Transactor {
}
}
+class ThrowBeforePartCommitTransactor implements Transactor {
+ private transactionCount = 0
+
+ constructor(private readonly delegate: Transactor) {}
+
+ async transaction<T>(
+ fn: (client: TxClient) => Promise<T>,
+ options?: TransactionOptions
+ ): Promise<T> {
+ this.transactionCount += 1
+ if (this.transactionCount === 2)
+ throw new CommitOutcomeUnknownError(new Error('synthetic pre-commit failure'))
+ return this.delegate.transaction(fn, options)
+ }
+}
+
async function settleWithin<T>(
promise: Promise<T>,
timeoutMs = 500
@@ -544,6 +569,43 @@ describe('GfsUploadSessionService', () => {
expect(db.parts.find(part => Number(part.part_number) === 1)?.state).toBe('reserved')
})
+ it('removes a rename-before-commit orphan while the reserved lease is still locked', async () => {
+ tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-orphan-'))
+ const db = new MemoryDb()
+ const tx = new ThrowBeforePartCommitTransactor({ transaction: async fn => fn(db) })
+ const created = await service(db).create({
+ ...principal,
+ operation: 'create',
+ parentRid: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ name: 'orphan.bin',
+ sizeBytes: TEST_CONFIG.preferredPartBytes,
+ idempotencyKey: '67676767-6767-4676-8676-676767676767',
+ })
+ const uploads = service(db, TEST_CONFIG, { tx })
+ const payload = Buffer.alloc(TEST_CONFIG.preferredPartBytes, 0x41)
+ const finalPath = join(tempRoot, '.uploads', created.session.uploadId, 'parts', '0.part')
+
+ await expect(
+ uploads.putPart(
+ created.session.uploadId,
+ partGeometry(
+ {
+ expectedBytes: TEST_CONFIG.preferredPartBytes,
+ partBytes: TEST_CONFIG.preferredPartBytes,
+ },
+ 0
+ ),
+ createSha(payload),
+ Readable.from([payload]) as never,
+ principal
+ )
+ ).rejects.toBeInstanceOf(CommitOutcomeUnknownError)
+
+ await expect(stat(finalPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ expect(db.parts[0]).toMatchObject({ state: 'failed' })
+ expect(db.sessions[0]).toMatchObject({ active_part_count: 0 })
+ })
+
it('cancels terminally and rejects a conflicting idempotency fingerprint', async () => {
tempRoot = await mkdtemp(join(tmpdir(), 'gfsc-upload-'))
const db = new MemoryDb()
diff --git a/gfs-controller/src/upload/uploadSession.ts b/gfs-controller/src/upload/uploadSession.ts
index c0d8e73f8..89d9ddd7b 100644
--- a/gfs-controller/src/upload/uploadSession.ts
+++ b/gfs-controller/src/upload/uploadSession.ts
@@ -1013,14 +1013,33 @@ export class GfsUploadSessionService {
if (reconciliation)
return { part: reconciliation.part, session: toReceipt(reconciliation.session) }
await removeIfPresent(stagingPath)
- if (finalPath && reconciliation !== undefined) await removeIfPresent(finalPath)
await this.deps.tx
.transaction(async client => {
+ // A definitive row lock separates a pre-commit failure from an
+ // outcome-unknown commit. Only a still-reserved row with the same
+ // lease may safely lose its computed final path; the lock fences a
+ // fresh reservation from racing that removal.
+ const currentPart = await client.query(
+ `SELECT state, lease_epoch FROM gfs_upload_parts
+ WHERE upload_id = $1 AND part_number = $2 FOR UPDATE`,
+ [uploadId, part.partNumber]
+ )
+ const current = currentPart.rows[0] as
+ | { state?: unknown; lease_epoch?: unknown }
+ | undefined
+ if (
+ !current ||
+ current.state !== 'reserved' ||
+ Number(current.lease_epoch) !== reserved.leaseEpoch
+ )
+ return
+ if (finalPath && !(await removeStrict(finalPath)))
+ throw new Error('could not remove an orphaned committed-part path')
const failedPart = await client.query(
`UPDATE gfs_upload_parts SET state = 'failed', committed_at = NULL
- WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved'
- RETURNING 1`,
- [uploadId, part.partNumber]
+ WHERE upload_id = $1 AND part_number = $2 AND state = 'reserved' AND lease_epoch = $3
+ RETURNING 1`,
+ [uploadId, part.partNumber, reserved.leaseEpoch]
)
// The commit transaction already decremented the durable counter when
// it reached PostgreSQL. Only a part that was still reserved needs a