Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): enforce quotas while streaming uploads

johaven8107d40750da
diff --git a/backend/src/app.bootstrap.ts b/backend/src/app.bootstrap.ts
index b612e044..204e9ec2 100644
--- a/backend/src/app.bootstrap.ts
+++ b/backend/src/app.bootstrap.ts
@@ -22,13 +22,13 @@ export async function appBootstrap(): Promise<NestFastifyApplication> {
ignoreTrailingSlash: true,
maxParamLength: 256
},
- bodyLimit: 26214400 /* 25 MB */
+ bodyLimit: 26214400 // 25 MB
})
const app: NestFastifyApplication = await NestFactory.create<NestFastifyApplication>(AppModule, fastifyAdapter, {
bufferLogs: true
})
- /* NestJS starts listening for shutdown hooks */
+ // NestJS starts listening for shutdown hooks
app.enableShutdownHooks()
/* Fastify instance */
@@ -41,8 +41,9 @@ export async function appBootstrap(): Promise<NestFastifyApplication> {
bootstrapWebDAV(app, fastifyInstance)
/* PARSER */
- // '*' body parser allow binary data as a stream (unlimited body size)
- fastifyInstance.addContentTypeParser('*', { bodyLimit: 0 }, (_req: FastifyRequest, _payload: FastifyRequest['raw'], done) => done(null))
+ // Keep unknown binary payloads available through req.raw.
+ // Fastify's bodyLimit still protects buffered parsers; upload consumers meter this raw stream.
+ fastifyInstance.addContentTypeParser('*', (_req: FastifyRequest, _payload: FastifyRequest['raw'], done) => done(null))
/* INTERCEPTORS */
app.useGlobalInterceptors(
diff --git a/backend/src/app.service.spec.ts b/backend/src/app.service.spec.ts
index 4f63f48b..54abb867 100644
--- a/backend/src/app.service.spec.ts
+++ b/backend/src/app.service.spec.ts
@@ -83,6 +83,7 @@ describe(AppService.name, () => {
expect(conf.logger.colorize).toBe(true)
const tmpSecretFile = path.join(os.tmpdir(), 'secret')
fs.writeFileSync(tmpSecretFile, 'fooBAR8888')
+ vi.spyOn(console, 'log').mockImplementation(() => undefined)
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
process.env[`${ENVIRONMENT_PREFIX}APPLICATIONS_FILES_ONLYOFFICE_SECRET`] = 'fooBAR'
process.env[`${ENVIRONMENT_PREFIX}LOGGER_STDOUT`] = 'false'
diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
index 92b892e3..f4c00860 100644
--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts
@@ -14,6 +14,7 @@ import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'
import { FILE_MODE } from '../../constants/operations'
import { FileEvent } from '../../events/file-events'
import { FileLockProps } from '../../interfaces/file-props.interface'
+import { FileError } from '../../models/file-error'
import { LockConflict } from '../../models/file-lock-error'
import { FilesLockManager } from '../../services/files-lock-manager.service'
import * as filesUtils from '../../utils/files'
@@ -319,7 +320,7 @@ describe(CollaboraOnlineManager.name, () => {
vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)
vi.spyOn(filesUtils, 'fileName').mockReturnValue('document.docx')
vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/document-unique.docx')
- vi.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)
+ vi.spyOn(filesUtils, 'writeUploadFromStream').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'copyFileContent').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)
})
@@ -347,7 +348,11 @@ describe(CollaboraOnlineManager.name, () => {
expect(result).toEqual({
LastModifiedTime: '2024-01-01T11:00:00.000Z'
})
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/document-unique.docx', mockRequest.raw)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/document-unique.docx',
+ mockRequest.raw,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesUtils.copyFileContent).toHaveBeenCalledWith('/tmp/document-unique.docx', mockSpace.realPath)
expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')
expect(emitSpy).toHaveBeenCalledWith('event', {
@@ -374,6 +379,36 @@ describe(CollaboraOnlineManager.name, () => {
vi.spyOn(fs, 'stat').mockResolvedValue({ mtime: new Date('2024-01-01T10:00:00.000Z') } as any)
await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST))
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')
+ })
+
+ it('should reject a known document size above the remaining quota before writing', async () => {
+ const mockRequest = {
+ user: mockUser,
+ space: { ...mockSpace, storageQuota: 10, storageUsage: 9 },
+ headers: { 'content-length': '2' },
+ raw: new Readable()
+ } as unknown as FastifyCollaboraOnlineSpaceRequest
+
+ await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('Storage quota exceeded', HttpStatus.INSUFFICIENT_STORAGE))
+
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')
+ })
+
+ it('should preserve the upload size error and remove the temporary file', async () => {
+ const mockRequest = {
+ user: mockUser,
+ space: mockSpace,
+ headers: {},
+ raw: new Readable()
+ } as unknown as FastifyCollaboraOnlineSpaceRequest
+
+ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, 'File size limit exceeded'))
+
+ await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('File size limit exceeded', HttpStatus.PAYLOAD_TOO_LARGE))
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')
+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()
})
it('should throw error when timestamp mismatch', async () => {
diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
index ae35d09c..e40e2ba4 100644
--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts
@@ -16,6 +16,7 @@ import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'
import { FILE_MODE } from '../../constants/operations'
import { FileLockOptions } from '../../interfaces/file-lock.interface'
import { FileLockProps } from '../../interfaces/file-props.interface'
+import { FileError } from '../../models/file-error'
import { LockConflict } from '../../models/file-lock-error'
import { FilesLockManager } from '../../services/files-lock-manager.service'
import {
@@ -28,7 +29,7 @@ import {
isPathIsDir,
removeFiles,
uniqueFilePathFromDir,
- writeFromStream
+ writeUploadFromStream
} from '../../utils/files'
import {
COLLABORA_APP_LOCK,
@@ -44,10 +45,12 @@ import type { CollaboraOnlineCheckFileInfo, FastifyCollaboraOnlineSpaceRequest,
import { API_COLLABORA_ONLINE_FILES } from './collabora-online.routes'
import { FileEvent } from '../../events/file-events'
import { ACTION } from '../../../../common/constants'
+import { createUploadStreamLimiter, parseContentLength } from '../../utils/upload-file'
@Injectable()
export class CollaboraOnlineManager {
private logger = new Logger(CollaboraOnlineManager.name)
+ private readonly maxUploadSize = configuration.applications.files.maxUploadSize
private readonly externalCollaboraOnlineServer = configuration.applications.files.editors.collabora.externalServer || null
private readonly expiration = convertHumanTimeToSeconds(configuration.auth.token.refresh.expiration)
@@ -116,33 +119,48 @@ export class CollaboraOnlineManager {
await this.checkSpace(req.space)
await this.checkTimeStampFromHeaders(req)
const tmpFilePath = await uniqueFilePathFromDir(path.join(os.tmpdir(), fileName(req.space.realPath)))
+ let contentLength: number | undefined
try {
- await writeFromStream(tmpFilePath, req.raw)
- } catch (e) {
- throw new Error(`unable to save document : ${e.message}`)
- }
- // try to verify the downloaded size
- const contentLength = parseInt(req.headers['content-length'], 10)
- if (!isNaN(contentLength) && contentLength !== 0) {
- const tmpFileSize = await fileSize(tmpFilePath)
- if (tmpFileSize !== contentLength) {
- this.logger.error({ tag: this.saveDocument.name, msg: `document size differs (${tmpFileSize} != ${contentLength})` })
- throw new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST)
+ try {
+ const fileLimiter = createUploadStreamLimiter(req.space, this.maxUploadSize).createFileLimiter()
+ contentLength = parseContentLength(req.headers['content-length'])
+ if (contentLength !== undefined) {
+ fileLimiter.assertKnownSize(contentLength)
+ }
+ await writeUploadFromStream(tmpFilePath, req.raw, { limiter: fileLimiter })
+ } catch (e) {
+ if (e instanceof FileError) {
+ throw new HttpException(e.message, e.httpCode)
+ }
+ throw new Error(`unable to save document : ${e.message}`)
+ }
+ // try to verify the downloaded size
+ if (contentLength !== undefined && contentLength !== 0) {
+ const tmpFileSize = await fileSize(tmpFilePath)
+ if (tmpFileSize !== contentLength) {
+ this.logger.error({ tag: this.saveDocument.name, msg: `document size differs (${tmpFileSize} != ${contentLength})` })
+ throw new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST)
+ }
+ } else if (contentLength === 0) {
+ this.logger.warn({ tag: this.saveDocument.name, msg: `content length is 0 : ${req.space.url}` })
+ }
+ // copy contents to avoid inode changes (dbFileHash in some cases)
+ try {
+ await copyFileContent(tmpFilePath, req.space.realPath)
+ // emit file event
+ FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })
+ const fStats = await fs.stat(req.space.realPath)
+ return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto
+ } catch (e) {
+ this.logger.error({ tag: this.saveDocument.name, msg: `unable to save document: ${e}` })
+ throw new HttpException('Unable to save document', HttpStatus.INTERNAL_SERVER_ERROR)
+ }
+ } finally {
+ try {
+ await removeFiles(tmpFilePath)
+ } catch (e) {
+ this.logger.error({ tag: this.saveDocument.name, msg: `failed to remove temporary document ${tmpFilePath}: ${e.message}` })
}
- } else if (contentLength === 0) {
- this.logger.warn({ tag: this.saveDocument.name, msg: `content length is 0 : ${req.space.url}` })
- }
- // copy contents to avoid inode changes (dbFileHash in some cases)
- try {
- await copyFileContent(tmpFilePath, req.space.realPath)
- // emit file event
- FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })
- await removeFiles(tmpFilePath)
- const fStats = await fs.stat(req.space.realPath)
- return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto
- } catch (e) {
- this.logger.error({ tag: this.saveDocument.name, msg: `unable to save document: ${e}` })
- throw new HttpException('Unable to save document', HttpStatus.INTERNAL_SERVER_ERROR)
}
}
diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
index c4101010..8d97fe45 100644
--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts
@@ -17,6 +17,7 @@ import { FILE_MODE } from '../../constants/operations'
import { FileEvent } from '../../events/file-events'
import { LockConflict } from '../../models/file-lock-error'
import { FilesLockManager } from '../../services/files-lock-manager.service'
+import { maxFileSizeExceededError } from '../../utils/errors'
import * as filesUtils from '../../utils/files'
import { OnlyOfficeManager } from './only-office-manager.service'
import { ONLY_OFFICE_APP_LOCK } from './only-office.constants'
@@ -260,7 +261,7 @@ describe(OnlyOfficeManager.name, () => {
filesLockManager.isPathLocked.mockResolvedValue(false)
cache.del.mockResolvedValue(true)
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValue('/tmp/temp-file.docx')
- vi.mocked(filesUtils.writeFromStream).mockResolvedValue(undefined)
+ vi.mocked(filesUtils.writeUploadFromStream).mockResolvedValue(undefined)
vi.mocked(filesUtils.fileSize).mockResolvedValue(12)
vi.mocked(filesUtils.copyFileContent).mockResolvedValue(undefined)
vi.mocked(filesUtils.removeFiles).mockResolvedValue(undefined)
@@ -299,6 +300,11 @@ describe(OnlyOfficeManager.name, () => {
source: 'editor'
})
expect(httpService.axiosRef).toHaveBeenCalledWith(expect.objectContaining({ url, maxRedirects: 0 }))
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ expect.stringContaining('sync-in-onlyoffice-'),
+ expect.anything(),
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
}
it('should allow internal container document downloads when external server is not configured', async () => {
@@ -404,6 +410,41 @@ describe(OnlyOfficeManager.name, () => {
expect(result.error).not.toBe(0)
})
+ it('should reject an oversized editor document and remove the temporary file', async () => {
+ jwtService.verifyAsync.mockResolvedValue({
+ status: 2,
+ actions: [],
+ notmodified: false,
+ url: mockDocumentUrl
+ })
+ mockSuccessfulDownload()
+ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(maxFileSizeExceededError())
+
+ const result = await service.callBack(mockUser, mockSpaceEnv, mockToken)
+
+ expect(result).toEqual({ error: 'unable to download document : File size limit exceeded' })
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(expect.stringContaining('sync-in-onlyoffice-'))
+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()
+ })
+
+ it('should reject an editor document above the remaining quota before writing', async () => {
+ const quotaSpace = { ...mockSpaceEnv, storageQuota: 12, storageUsage: 1 } as unknown as SpaceEnv
+ jwtService.verifyAsync.mockResolvedValue({
+ status: 2,
+ actions: [],
+ notmodified: false,
+ url: mockDocumentUrl
+ })
+ mockSuccessfulDownload()
+
+ const result = await service.callBack(mockUser, quotaSpace, mockToken)
+
+ expect(result).toEqual({ error: 'unable to download document : Storage quota exceeded' })
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(expect.stringContaining('sync-in-onlyoffice-'))
+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()
+ })
+
it('should reject document downloads from unexpected hosts when external server is configured', async () => {
;(service as any).externalOnlyOfficeServer = 'http://localhost:3000/onlyoffice'
jwtService.verifyAsync.mockResolvedValue({
diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
index 072408e5..c2911144 100644
--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts
@@ -34,7 +34,7 @@ import {
isPathExists,
isPathIsDir,
removeFiles,
- writeFromStream
+ writeUploadFromStream
} from '../../utils/files'
import {
EURO_OFFICE_APP_LOCK,
@@ -51,10 +51,12 @@ import { OnlyOfficeCallBack, OnlyOfficeConfig, OnlyOfficeConvertForm } from './o
import { API_ONLY_OFFICE_CALLBACK, API_ONLY_OFFICE_DOCUMENT } from './only-office.routes'
import { FileEvent } from '../../events/file-events'
import { ACTION } from '../../../../common/constants'
+import { createUploadStreamLimiter, parseContentLength } from '../../utils/upload-file'
@Injectable()
export class OnlyOfficeManager {
private logger = new Logger(OnlyOfficeManager.name)
+ private readonly maxUploadSize = configuration.applications.files.maxUploadSize
private readonly officeConfig = configuration.applications.files.editors.onlyoffice.enabled
? configuration.applications.files.editors.onlyoffice
: configuration.applications.files.editors.eurooffice
@@ -350,17 +352,25 @@ export class OnlyOfficeManager {
}
// download file
- let res: AxiosResponse
+ let downloadStream: AxiosResponse['data'] | undefined
+ let contentLength: number | undefined
try {
- res = await this.http.axiosRef({
+ const res: AxiosResponse = await this.http.axiosRef({
method: HTTP_METHOD.GET,
url: finalDownloadUrl,
responseType: 'stream',
maxRedirects: 0,
httpsAgent: new https.Agent({ rejectUnauthorized: this.rejectUnauthorized })
})
- await writeFromStream(tmpFilePath, res.data)
+ downloadStream = res.data
+ const fileLimiter = createUploadStreamLimiter(space, this.maxUploadSize).createFileLimiter()
+ contentLength = parseContentLength(res.headers['content-length'])
+ if (contentLength !== undefined) {
+ fileLimiter.assertKnownSize(contentLength)
+ }
+ await writeUploadFromStream(tmpFilePath, downloadStream, { limiter: fileLimiter })
} catch (e) {
+ downloadStream?.destroy?.()
this.logger.error({
tag: this.saveDocument.name,
msg: `failed to download document from ${finalParsedDownloadUrl.origin}${finalParsedDownloadUrl.pathname} : ${space.url} - ${e.message}`
@@ -369,8 +379,7 @@ export class OnlyOfficeManager {
}
// try to verify the downloaded size
- const contentLength = Number(res.headers['content-length'])
- if (!isNaN(contentLength) && contentLength !== 0) {
+ if (contentLength !== undefined && contentLength !== 0) {
const tmpFileSize = await fileSize(tmpFilePath)
if (tmpFileSize !== contentLength) {
throw new Error(`document size differs (${tmpFileSize} != ${contentLength})`)
diff --git a/backend/src/applications/files/interfaces/save-stream.interface.ts b/backend/src/applications/files/interfaces/save-stream.interface.ts
index d2deb6ba..e2c81527 100644
--- a/backend/src/applications/files/interfaces/save-stream.interface.ts
+++ b/backend/src/applications/files/interfaces/save-stream.interface.ts
@@ -9,6 +9,9 @@ export interface SaveStreamTmpFileValidationContext {
export interface SaveStreamOptions {
dav?: { depth: LOCK_DEPTH; lockTokens: string[] }
checksumAlg?: string
+ // Expected final file size, including any resumed range.
+ expectedUploadSize?: number
+ maxSize?: number
tmpPath?: string
validateTmpFile?: (ctx: SaveStreamTmpFileValidationContext) => Promise<void>
}
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 4a3cfb79..0fa61536 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -6,6 +6,7 @@ import path from 'node:path'
import { Readable } from 'node:stream'
import { transformAndValidate } from '../../../common/functions'
import * as imageUtils from '../../../common/image'
+import { configuration } from '../../../configuration/config.environment'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
import { NotificationsManager } from '../../notifications/services/notifications-manager.service'
import { SpacesManager } from '../../spaces/services/spaces-manager.service'
@@ -95,8 +96,8 @@ describe(FilesManager.name, () => {
})
const expectNoWriteOperations = () => {
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
- expect(filesUtils.writeFromStreamAndChecksum).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStreamAndChecksum).not.toHaveBeenCalled()
expect(filesUtils.makeDir).not.toHaveBeenCalled()
expect(filesUtils.createEmptyFile).not.toHaveBeenCalled()
expect(filesUtils.copyFileContent).not.toHaveBeenCalled()
@@ -185,8 +186,8 @@ describe(FilesManager.name, () => {
vi.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)
vi.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')
vi.spyOn(filesUtils, 'tempFilePath').mockReturnValue('/tmp/staged-file')
- vi.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)
- vi.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')
+ vi.spyOn(filesUtils, 'writeUploadFromStream').mockResolvedValue(undefined)
+ vi.spyOn(filesUtils, 'writeUploadFromStreamAndChecksum').mockResolvedValue('sha256-abc')
vi.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'copyFiles').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)
@@ -243,11 +244,55 @@ describe(FilesManager.name, () => {
expect(result).toBe(false)
expect(filesLockManager.create).toHaveBeenCalledWith(user, space.dbFile, 'Sync-in', DEPTH.RESOURCE)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(space.realPath, expect.anything(), 0)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ space.realPath,
+ expect.anything(),
+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })
+ )
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: space.realPath })
})
+ it('should short-circuit a known body above a stricter max size supplied by the caller', async () => {
+ const space = makeSpace()
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)
+
+ await expect(
+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-length': '5' }, raw: Readable.from(['hello']) } as any, {
+ maxSize: 4
+ })
+ ).rejects.toEqual(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))
+
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesLockManager.create).not.toHaveBeenCalled()
+ })
+
+ it('should stop an upload without content-length when streamed bytes exceed the remaining quota', async () => {
+ const space = makeSpace({ storageQuota: 10, storageUsage: 9 })
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)
+ vi.mocked(filesUtils.writeUploadFromStream).mockImplementationOnce(async (_path, _stream, options) => {
+ options.limiter.consume(2)
+ })
+
+ await expect(service.saveStream(user, space, { method: 'PUT', headers: {}, raw: Readable.from(['xx']) } as any)).rejects.toEqual(
+ new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)
+ )
+
+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
+ })
+
+ it('should short-circuit a known body above the remaining quota', async () => {
+ const space = makeSpace({ storageQuota: 10, storageUsage: 9 })
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)
+
+ await expect(
+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-length': '2' }, raw: Readable.from(['xx']) } as any)
+ ).rejects.toEqual(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
+
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesLockManager.create).not.toHaveBeenCalled()
+ })
+
it('should use DAV conflict checks and checksum mode when requested', async () => {
const space = makeSpace()
setPathExists({ [space.realPath]: true, [path.dirname(space.realPath)]: true }, true)
@@ -262,7 +307,70 @@ describe(FilesManager.name, () => {
expect(checksum).toBe('sha256-abc')
expect(filesLockManager.checkConflicts).toHaveBeenCalledWith(space.dbFile, DEPTH.RESOURCE, { userId: 7, lockTokens: ['token'] })
expect(filesLockManager.create).not.toHaveBeenCalled()
- expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStreamAndChecksum).toHaveBeenCalledWith(
+ space.realPath,
+ expect.anything(),
+ 'sha256',
+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 100, consume: expect.any(Function) }) })
+ )
+ })
+
+ it('should accept a content range starting at zero for a new file', async () => {
+ const space = makeSpace()
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)
+
+ const result = await service.saveStream(
+ user,
+ space,
+ { method: 'PUT', headers: { 'content-range': 'bytes 0-4/5' }, raw: Readable.from(['hello']) } as any,
+ { dav: { depth: DEPTH.RESOURCE, lockTokens: [] } }
+ )
+
+ expect(result).toBe(false)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ space.realPath,
+ expect.anything(),
+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })
+ )
+ })
+
+ it('should interpret the expected upload size as the final size of a resumed upload', async () => {
+ const space = makeSpace({ storageQuota: 10, storageUsage: 8 })
+ const tmpPath = '/data/users/john/tmp/sync-in-file.txt'
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true, [tmpPath]: true }, false)
+ vi.mocked(filesUtils.fileSize).mockResolvedValueOnce(8)
+
+ await expect(
+ service.saveStream(
+ user,
+ space,
+ {
+ method: 'PUT',
+ headers: { 'content-range': 'bytes 8-9/10', 'content-length': '2' },
+ raw: Readable.from(['xx'])
+ } as any,
+ { tmpPath, expectedUploadSize: 10 }
+ )
+ ).resolves.toBe(false)
+
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ tmpPath,
+ expect.anything(),
+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 8, consume: expect.any(Function) }) })
+ )
+ })
+
+ it('should reject a non-zero content range when the file does not exist', async () => {
+ const space = makeSpace()
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)
+
+ await expect(
+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-range': 'bytes 5-9/10' }, raw: Readable.from(['hello']) } as any, {
+ dav: { depth: DEPTH.RESOURCE, lockTokens: [] }
+ })
+ ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size'))
+
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
})
it('should validate tmp stream before moving it to the destination', async () => {
@@ -282,7 +390,12 @@ describe(FilesManager.name, () => {
).rejects.toEqual(validationError)
expect(validateTmpFile).toHaveBeenCalledWith({ tmpPath, realPath: space.realPath, checksum: 'sha256-abc' })
- expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalledWith(tmpPath, expect.anything(), 0, 'sha256')
+ expect(filesUtils.writeUploadFromStreamAndChecksum).toHaveBeenCalledWith(
+ tmpPath,
+ expect.anything(),
+ 'sha256',
+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })
+ )
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(tmpPath)
expect(emitSpy).not.toHaveBeenCalled()
@@ -359,11 +472,15 @@ describe(FilesManager.name, () => {
await service.saveMultipart(user, space, req as any)
- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string
+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string
expect(filesLockManager.createOrRefresh).toHaveBeenCalled()
expect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)
- expect(tmpWritePath.endsWith('-report.txt')).toBe(true)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, expect.anything())
+ expect(path.basename(tmpWritePath)).toMatch(/^report\.txt-upload-/)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ tmpWritePath,
+ expect.anything(),
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, '/data/users/john/files/report.txt', true)
expect(emitSpy).toHaveBeenCalledWith(
'event',
@@ -390,7 +507,7 @@ describe(FilesManager.name, () => {
await expect(service.saveMultipart(user, space, req as any)).rejects.toEqual(new FileError(HttpStatus.NOT_FOUND, 'Location not found'))
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()
@@ -413,9 +530,13 @@ describe(FilesManager.name, () => {
await service.saveMultipart(user, space, req as any)
- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string
+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string
expect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ tmpWritePath,
+ file,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, space.realPath, true)
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: space.realPath }))
@@ -451,11 +572,42 @@ describe(FilesManager.name, () => {
await service.saveMultipart(user, space, req as any)
expect(filesUtils.makeDir).toHaveBeenCalledWith(dstDir, true)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(dstFile, file)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ dstFile,
+ file,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-created')
expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.ADD, rPath: dstFile }))
})
+ it('should share the remaining quota across multipart files', async () => {
+ const space = makeSpace({ storageQuota: 5, storageUsage: 0 })
+ const parentPath = path.dirname(space.realPath)
+ const firstPath = path.join(parentPath, 'one.bin')
+ const secondPath = path.join(parentPath, 'two.bin')
+ setPathExists({ [space.realPath]: false, [parentPath]: true, [firstPath]: false, [secondPath]: false }, false)
+ vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p === parentPath)
+ vi.mocked(filesUtils.writeUploadFromStream).mockImplementation(async (_path, _stream, options) => {
+ options.limiter.consume(3)
+ })
+
+ const req = {
+ method: 'POST',
+ files: async function* () {
+ yield { filename: 'one.bin', file: Readable.from(['one']) }
+ yield { filename: 'two.bin', file: Readable.from(['two']) }
+ }
+ }
+
+ await expect(service.saveMultipart(user, space, req as any)).rejects.toEqual(
+ new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)
+ )
+
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledTimes(2)
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(secondPath)
+ })
+
it('should reject POST when resolved multipart destination already exists', async () => {
const space = makeSpace()
const partFileName = 'folder/file.txt'
@@ -484,7 +636,7 @@ describe(FilesManager.name, () => {
new FileError(HttpStatus.METHOD_NOT_ALLOWED, 'Resource already exists')
)
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()
@@ -510,7 +662,7 @@ describe(FilesManager.name, () => {
expect(filesUtils.isPathExists).not.toHaveBeenCalledWith(forbiddenFile)
expect(filesUtils.isPathIsDir).not.toHaveBeenCalledWith(forbiddenFile)
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()
expect(emitSpy).not.toHaveBeenCalled()
@@ -535,9 +687,13 @@ describe(FilesManager.name, () => {
new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)
)
- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string
+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string
expect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ tmpWritePath,
+ file,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpWritePath)
expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(space.realPath)
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
@@ -565,7 +721,11 @@ describe(FilesManager.name, () => {
)
expect(req.files).toHaveBeenCalled()
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(dstFile, file)
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ dstFile,
+ file,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(filesUtils.removeFiles).toHaveBeenCalledWith(dstFile)
expect(emitSpy).not.toHaveBeenCalled()
})
@@ -588,7 +748,7 @@ describe(FilesManager.name, () => {
new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)
)
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
})
it('should not map non-file-size multipart 413 errors to file size limit', async () => {
@@ -607,7 +767,7 @@ describe(FilesManager.name, () => {
await expect(service.saveMultipart(user, space, req as any)).rejects.toBe(error)
- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
})
it.each([
@@ -649,13 +809,17 @@ describe(FilesManager.name, () => {
await expect(service.saveMultipart(user, space, req as any)).rejects.toBe(error)
- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)
+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
+ tmpWritePath,
+ file,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(deleteSpy).toHaveBeenCalledTimes(1)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, expect.stringContaining(path.basename(partFileName)), true)
expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpWritePath)
expect(emitSpy).not.toHaveBeenCalled()
- expect(vi.mocked(filesUtils.writeFromStream).mock.invocationCallOrder[0]).toBeLessThan(deleteSpy.mock.invocationCallOrder[0])
+ expect(vi.mocked(filesUtils.writeUploadFromStream).mock.invocationCallOrder[0]).toBeLessThan(deleteSpy.mock.invocationCallOrder[0])
expect(deleteSpy.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.moveFiles).mock.invocationCallOrder[0])
})
@@ -685,7 +849,7 @@ describe(FilesManager.name, () => {
await service.saveMultipart(user, space, req as any)
- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string
+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string
expect(deleteSpy).toHaveBeenCalledTimes(1)
expect(filesUtils.makeDir).toHaveBeenCalledWith(dstDir, true)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, dstFile, true)
@@ -1070,13 +1234,10 @@ describe(FilesManager.name, () => {
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/download.txt')
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(
+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(
taskPath('task-1', 'download.txt'),
expect.anything(),
- 0,
- 55,
- undefined,
- expect.any(Function)
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }), onProgress: expect.any(Function) })
)
expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-1', 'download.txt'), '/tmp/download.txt')
@@ -1090,12 +1251,32 @@ describe(FilesManager.name, () => {
expect(fileEmitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })
})
+ it('should reject a remote download exceeding maxUploadSize before GET', async () => {
+ const space = makeSpace()
+ const tmpPath = '/data/users/john/tmp/download.txt-download-uuid'
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/download.txt')
+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce(tmpPath)
+ http.axiosRef.mockResolvedValueOnce({
+ headers: { 'content-length': `${configuration.applications.files.maxUploadSize + 1}` },
+ request: { socket: { remoteAddress: '8.8.8.8' } }
+ })
+
+ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toEqual(
+ new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)
+ )
+
+ expect(http.axiosRef).toHaveBeenCalledTimes(1)
+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpPath)
+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
+ })
+
it('should cleanup partial file and skip ADD event when download write fails', async () => {
const error = new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)
const space = makeSpace()
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/download.txt')
vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')
- vi.mocked(filesUtils.writeFromStream).mockRejectedValueOnce(error)
+ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(error)
http.axiosRef
.mockResolvedValueOnce({
headers: { 'content-length': '55' },
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index c9dc8fc0..38c98e38 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -5,6 +5,7 @@ import { Readable } from 'node:stream'
import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'
import { generateThumbnail } from '../../../common/image'
import { SERVER_NAME } from '../../../common/shared'
+import { configuration } from '../../../configuration/config.environment'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
import { HTTP_METHOD } from '../../applications.constants'
import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/constants/notifications'
@@ -47,11 +48,12 @@ import {
moveFiles,
removeFiles,
tempFilePath,
+ temporaryPathPrefix,
touchFile,
uniqueDatedFilePath,
uniqueFilePathFromDir,
- writeFromStream,
- writeFromStreamAndChecksum
+ writeUploadFromStream,
+ writeUploadFromStreamAndChecksum
} from '../utils/files'
import { SendFile } from '../utils/send-file'
import { extractZip } from '../utils/unzip-file'
@@ -61,7 +63,13 @@ import { FilesLockManager } from './files-lock-manager.service'
import { FilesQueries } from './files-queries.service'
import { FileEvent, FileTaskEvent } from '../events/file-events'
import { ACTION } from '../../../common/constants'
-import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'
+import {
+ createUploadStreamLimiter,
+ isMultipartFileTooLargeError,
+ parseContentLength,
+ parseContentRange,
+ uploadTmpFilePath
+} from '../utils/upload-file'
import { maxFileSizeExceededError } from '../utils/errors'
import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'
import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'
@@ -71,7 +79,7 @@ import { FILE_ERROR } from '../constants/errors'
@Injectable()
export class FilesManager {
- /* Spaces permissions are checked in the space guard, except for the copy/move destination */
+ // Spaces permissions are checked in the space guard, except for the copy/move destination
private logger = new Logger(FilesManager.name)
constructor(
@@ -107,13 +115,41 @@ export class FilesManager {
if (fExists && (await isPathIsDir(space.realPath))) {
throw new FileError(HttpStatus.METHOD_NOT_ALLOWED, 'The location is a directory')
}
+
+ // Resolve the final file-size boundary and reject a known oversized body
+ // before creating directories or acquiring a lock.
+ let startRange = 0
+ const contentRange = req.headers['content-range']
+ if (contentRange !== undefined) {
+ // With PUT method, some webdav clients use the `content-range` header,
+ // which is normally reserved for a response to a request containing the `range` header.
+ // However, for more compatibility let's accept it.
+ startRange = parseContentRange(contentRange).start
+ // Ranged PUTs are append-only resumes. A missing file has size zero, so it can only
+ // be created by a first range starting at offset zero.
+ const currentSize = fExists || fTmpExists ? await fileSize(options?.tmpPath || space.realPath) : 0
+ if (startRange !== currentSize) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size')
+ }
+ }
+ const maxUploadSize = configuration.applications.files.maxUploadSize
+ const maxSize = Math.min(maxUploadSize, options?.maxSize ?? maxUploadSize)
+ const fileLimiter = createUploadStreamLimiter(space, maxSize).createFileLimiter(startRange)
+ const contentLength = parseContentLength(req.headers['content-length'])
+ if (contentLength !== undefined) {
+ fileLimiter.assertKnownSize(contentLength)
+ }
+ if (options?.expectedUploadSize !== undefined) {
+ fileLimiter.assertFinalSize(options.expectedUploadSize)
+ }
+
if (options?.tmpPath) {
// Ensure tmpPath parent dir exists
await makeDir(dirName(options.tmpPath), true)
} else if (!(await isPathExists(dirName(space.realPath)))) {
throw new FileError(HttpStatus.CONFLICT, 'Parent must exists')
}
- /* File Lock */
+ // File Lock
let fileLock: FileLock | undefined
if (options?.dav) {
// Check locks
@@ -132,29 +168,17 @@ export class FilesManager {
const fileEventAction = fExists ? ACTION.UPDATE : ACTION.ADD
let fileWritten = false
try {
- // Check range
- let startRange = 0
- if ((fExists || fTmpExists) && req.headers['content-range']) {
- // With PUT method, some webdav clients use the `content-range` header,
- // which is normally reserved for a response to a request containing the `range` header.
- // However, for more compatibility let's accept it.
- const match = /\d+/.exec(req.headers['content-range'])
- if (!match.length) {
- throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')
- }
- startRange = parseInt(match[0], 10)
- const size = await fileSize(options?.tmpPath || space.realPath)
- if (startRange !== size) {
- throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size')
- }
- }
// todo: check file in db to update
// todo : versioning here
let checksum: string
if (options?.checksumAlg) {
- checksum = await writeFromStreamAndChecksum(options?.tmpPath || space.realPath, req.raw, startRange, options.checksumAlg)
+ checksum = await writeUploadFromStreamAndChecksum(options?.tmpPath || space.realPath, req.raw, options.checksumAlg, {
+ limiter: fileLimiter
+ })
} else {
- await writeFromStream(options?.tmpPath || space.realPath, req.raw, startRange)
+ await writeUploadFromStream(options?.tmpPath || space.realPath, req.raw, {
+ limiter: fileLimiter
+ })
}
if (options?.tmpPath) {
await options.validateTmpFile?.({ tmpPath: options.tmpPath, realPath: space.realPath, checksum })
@@ -205,6 +229,7 @@ export class FilesManager {
const patchMethod = req.method === HTTP_METHOD.PATCH
const postMethod = req.method === HTTP_METHOD.POST
const realParentPath = dirName(space.realPath)
+ const uploadLimiter = createUploadStreamLimiter(space, configuration.applications.files.maxUploadSize)
// For POST, space.realPath can be either the final file path or the root directory for a folder upload.
if (postMethod && (await isPathExists(space.realPath))) {
@@ -272,7 +297,8 @@ export class FilesManager {
let fileWritten = false
// Do
try {
- await writeFromStream(writePath, part.file)
+ const fileLimiter = uploadLimiter.createFileLimiter()
+ await writeUploadFromStream(writePath, part.file, { limiter: fileLimiter })
// With throwFileSizeLimit disabled, multipart marks the file stream as truncated instead of rejecting.
if (part.file.truncated) {
throw maxFileSizeExceededError()
@@ -456,7 +482,7 @@ export class FilesManager {
const isDir = await isPathIsDir(srcSpace.realPath)
if (dstSpace.storageQuota) {
- /* Skip validation when moving to the same space; for copy operations, run all checks. */
+ // Skip validation when moving to the same space; for copy operations, run all checks.
if (!isMove || (isMove && srcSpace.id !== dstSpace.id)) {
const size = isDir ? (await dirSize(srcSpace.realPath))[0] : await fileSize(srcSpace.realPath)
if (dstSpace.willExceedQuota(size)) {
@@ -602,7 +628,7 @@ export class FilesManager {
const dstPath = await uniqueFilePathFromDir(space.realPath)
const tmpPath = isTaskContext
? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)
- : tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)
+ : tempFilePath(user.tmpPath, temporaryPathPrefix(dstPath, 'download'))
const dbFile = space.dbFile
dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))
@@ -616,6 +642,7 @@ export class FilesManager {
await new DownloadFile(this.http).download(downloadDto, tmpPath, {
space,
publishedPath: dstPath,
+ maxSize: configuration.applications.files.maxUploadSize,
signal,
onProgress: isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined
})
@@ -646,7 +673,7 @@ export class FilesManager {
const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))
const tmpPath = isTaskContext
? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)
- : tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)
+ : tempFilePath(user.tmpPath, temporaryPathPrefix(dstPath, 'compress'))
// create lock
let fileLock: FileLock | undefined
if (dto.compressInDirectory) {
@@ -701,7 +728,7 @@ export class FilesManager {
const dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))
const tmpPath = isTaskContext
? await createTaskTemporaryDir(user.tasksPath, space.task!.cacheKey, dstPath)
- : await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)
+ : await makeTempDir(user.tmpPath, temporaryPathPrefix(dstPath, 'extract'))
let fileLock: FileLock | undefined
try {
// create lock
diff --git a/backend/src/applications/files/utils/download-file.spec.ts b/backend/src/applications/files/utils/download-file.spec.ts
index 566a0150..08b3b6ad 100644
--- a/backend/src/applications/files/utils/download-file.spec.ts
+++ b/backend/src/applications/files/utils/download-file.spec.ts
@@ -1,16 +1,17 @@
import { lookup } from 'node:dns/promises'
import { HttpService } from '@nestjs/axios'
-import { HttpStatus } from '@nestjs/common'
+import { HttpStatus, Logger } from '@nestjs/common'
import { Readable } from 'node:stream'
import { HTTP_METHOD } from '../../applications.constants'
import { FileError } from '../models/file-error'
-import { writeFromStream } from './files'
+import { writeUploadFromStream } from './files'
import { DownloadFile } from './download-file'
import type { Mock } from 'vitest'
import { FILE_ERROR } from '../constants/errors'
vi.mock('./files', () => ({
- writeFromStream: vi.fn()
+ writeUploadFromStream: vi.fn(),
+ temporaryPathPrefix: vi.fn()
}))
vi.mock('node:dns/promises', () => ({
lookup: vi.fn()
@@ -44,13 +45,15 @@ describe(DownloadFile.name, () => {
})
beforeEach(() => {
+ vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined)
http = { axiosRef: vi.fn() }
lookupMock.mockResolvedValue([{ address: '8.8.8.8', family: 4 }])
- vi.mocked(writeFromStream).mockResolvedValue(undefined)
+ vi.mocked(writeUploadFromStream).mockResolvedValue(undefined)
})
afterEach(() => {
vi.clearAllMocks()
+ vi.restoreAllMocks()
})
it.each(blockedRemoteAddresses)('rejects blocked remote address "%s" on HEAD by default', async (remoteAddress) => {
@@ -64,7 +67,7 @@ describe(DownloadFile.name, () => {
expect(http.axiosRef).toHaveBeenCalledWith(
expect.objectContaining({ method: HTTP_METHOD.HEAD, url: 'https://example.test/file.txt', maxRedirects: 0 })
)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('rejects blocked DNS resolutions before calling HEAD', async () => {
@@ -76,7 +79,7 @@ describe(DownloadFile.name, () => {
expect(lookupMock).toHaveBeenCalledWith('example.test', { all: true, order: 'verbatim' })
expect(http.axiosRef).not.toHaveBeenCalled()
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('rejects mixed DNS resolutions when one address is blocked', async () => {
@@ -90,7 +93,7 @@ describe(DownloadFile.name, () => {
).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))
expect(http.axiosRef).not.toHaveBeenCalled()
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('follows a safe redirect manually', async () => {
@@ -115,7 +118,7 @@ describe(DownloadFile.name, () => {
2,
expect.objectContaining({ method: HTTP_METHOD.HEAD, url: 'https://cdn.example.test/file.txt', maxRedirects: 0 })
)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('rejects a redirect to a blocked DNS resolution before following it', async () => {
@@ -127,7 +130,7 @@ describe(DownloadFile.name, () => {
).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))
expect(http.axiosRef).toHaveBeenCalledTimes(1)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('rejects a missing remote address on HEAD by default', async () => {
@@ -138,7 +141,7 @@ describe(DownloadFile.name, () => {
).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))
expect(http.axiosRef).toHaveBeenCalledTimes(1)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it.each(publicRemoteAddresses)('allows public remote address "%s" on HEAD by default', async (remoteAddress) => {
@@ -161,7 +164,7 @@ describe(DownloadFile.name, () => {
httpsAgent: expect.any(Object)
})
)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('uses a guarded agent lookup for connection-time resolutions', async () => {
@@ -187,7 +190,7 @@ describe(DownloadFile.name, () => {
new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', { getContentInfo: true })
).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('allows private IPs on HEAD when allowPrivateIP is enabled for content info', async () => {
@@ -210,7 +213,7 @@ describe(DownloadFile.name, () => {
lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT'
})
expect(http.axiosRef).toHaveBeenCalledTimes(1)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('rejects private IPs on GET by default and closes the stream', async () => {
@@ -225,7 +228,7 @@ describe(DownloadFile.name, () => {
).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))
expect(destroySpy).toHaveBeenCalled()
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it.each(['', '-1', '1.5', 'Infinity', '9007199254740992', 'abc'])('rejects invalid content-length "%s"', async (contentLength) => {
@@ -236,7 +239,18 @@ describe(DownloadFile.name, () => {
).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))
expect(http.axiosRef).toHaveBeenCalledTimes(1)
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
+ })
+
+ it('rejects a missing content-length when no explicit maximum is available', async () => {
+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8'))
+
+ await expect(
+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')
+ ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))
+
+ expect(http.axiosRef).toHaveBeenCalledTimes(1)
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('allows missing content-length when maxSize is provided', async () => {
@@ -247,27 +261,30 @@ describe(DownloadFile.name, () => {
maxSize: 1024
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
})
- it('rejects missing content-length for space downloads even when maxSize is provided', async () => {
- const space = { willExceedQuota: vi.fn() }
- http.axiosRef.mockResolvedValueOnce(response('8.8.8.8'))
+ it('allows missing content-length for space downloads when maxSize is provided', async () => {
+ const stream = Readable.from(['abc'])
+ const space = { storageQuota: 10, storageUsage: 7 }
+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8')).mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })
- await expect(
- new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {
- space: space as any,
- maxSize: 1024
- })
- ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))
+ await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {
+ space: space as any,
+ maxSize: 1024
+ })
- expect(space.willExceedQuota).not.toHaveBeenCalled()
- expect(writeFromStream).not.toHaveBeenCalled()
+ const options = vi.mocked(writeUploadFromStream).mock.calls[0][2]
+ expect(() => options.limiter.consume(4)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
})
- it('keeps content-length as the stream guard for space downloads even when maxSize is provided', async () => {
+ it('accepts a known content-length within the remaining quota', async () => {
const stream = Readable.from(['abc'])
- const space = { willExceedQuota: vi.fn().mockReturnValue(false) }
+ const space = { storageQuota: 100, storageUsage: 88 }
http.axiosRef
.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' }))
.mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })
@@ -277,8 +294,41 @@ describe(DownloadFile.name, () => {
maxSize: 1024
})
- expect(space.willExceedQuota).toHaveBeenCalledWith(12)
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
+ })
+
+ it('rejects a known content-length above the remaining quota before GET', async () => {
+ const space = { storageQuota: 100, storageUsage: 89 }
+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' }))
+
+ await expect(
+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {
+ space: space as any,
+ maxSize: 1024
+ })
+ ).rejects.toEqual(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
+
+ expect(http.axiosRef).toHaveBeenCalledTimes(1)
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
+ })
+
+ it('rejects a space download exceeding maxSize before starting the GET request', async () => {
+ const space = { storageQuota: null, storageUsage: 0 }
+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '1025' }))
+
+ await expect(
+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {
+ space: space as any,
+ maxSize: 1024
+ })
+ ).rejects.toEqual(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))
+
+ expect(http.axiosRef).toHaveBeenCalledTimes(1)
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('forwards the progress callback to the stream writer', async () => {
@@ -292,7 +342,11 @@ describe(DownloadFile.name, () => {
onProgress
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, onProgress)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }), onProgress })
+ )
})
it('allows zero content-length and guards the written stream at zero bytes', async () => {
@@ -303,7 +357,11 @@ describe(DownloadFile.name, () => {
await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0, undefined, undefined)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
})
it('rejects redirects on GET after the HEAD URL has been resolved', async () => {
@@ -319,7 +377,7 @@ describe(DownloadFile.name, () => {
expect(http.axiosRef).toHaveBeenCalledTimes(2)
expect(destroySpy).toHaveBeenCalled()
- expect(writeFromStream).not.toHaveBeenCalled()
+ expect(writeUploadFromStream).not.toHaveBeenCalled()
})
it('allows private IPs on GET when allowPrivateIP is enabled', async () => {
@@ -332,7 +390,11 @@ describe(DownloadFile.name, () => {
allowPrivateIP: true
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
expect(http.axiosRef).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
@@ -357,6 +419,10 @@ describe(DownloadFile.name, () => {
maxSize: 1024
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)
+ expect(writeUploadFromStream).toHaveBeenCalledWith(
+ '/tmp/file.txt',
+ stream,
+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })
+ )
})
})
diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.ts
index 5965c397..105ff4ec 100644
--- a/backend/src/applications/files/utils/download-file.ts
+++ b/backend/src/applications/files/utils/download-file.ts
@@ -4,7 +4,7 @@ import { Agent as HttpsAgent } from 'node:https'
import type { LookupAddress } from 'node:dns'
import type { LookupFunction } from 'node:net'
import type { HttpService } from '@nestjs/axios'
-import { HttpStatus } from '@nestjs/common'
+import { HttpStatus, Logger } from '@nestjs/common'
import { AxiosHeaders, type AxiosRequestConfig, type AxiosResponse } from 'axios'
import ipaddr from 'ipaddr.js'
import { HTTP_METHOD } from '../../applications.constants'
@@ -13,15 +13,19 @@ import type { DownloadFileDto } from '../dto/file-operations.dto'
import { FileTaskEvent } from '../events/file-events'
import type { DownloadFileContentInfo, DownloadFileOptions } from '../interfaces/download-file.interface'
import { FileError } from '../models/file-error'
-import { writeFromStream } from './files'
+import { writeUploadFromStream } from './files'
import { FILE_ERROR } from '../constants/errors'
+import { createUploadStreamLimiter } from './upload-file'
interface DownloadFileRequestOptions {
allowPrivateIP?: boolean
maxRedirects?: number
}
+type DownloadStage = 'head' | 'validation' | 'get' | 'stream'
+
export class DownloadFile {
+ private readonly logger = new Logger(DownloadFile.name)
private static readonly dnsOptions = { all: true, order: 'verbatim' } as const
private static readonly maxRedirects = 1
private static readonly redirectStatus = new Set([301, 302, 303, 307, 308])
@@ -41,36 +45,76 @@ export class DownloadFile {
async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions & { getContentInfo?: false | undefined }): Promise<void>
async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions): Promise<void | DownloadFileContentInfo> {
const identityEncodingConfig = { decompress: false, headers: { 'Accept-Encoding': 'identity' } }
+ let stage: DownloadStage = 'head'
+ let currentUrl = downloadDto.url
+ let contentLength: number | null | undefined
+ let maxSize: number | undefined
+
+ try {
+ const { response: headRes, url } = await this.request(
+ currentUrl,
+ { method: HTTP_METHOD.HEAD, signal: options?.signal, ...identityEncodingConfig },
+ { allowPrivateIP: options?.allowPrivateIP }
+ )
+ currentUrl = url
+
+ stage = 'validation'
+ const headers = AxiosHeaders.from(headRes.headers)
+ contentLength = this.contentLength(headers)
+ if (options?.getContentInfo) {
+ return {
+ contentLength,
+ contentType: `${headers.getContentType()}`,
+ lastModified: headers.get('last-modified') as string | undefined
+ } satisfies DownloadFileContentInfo
+ }
- const { response: headRes, url } = await this.request(
- downloadDto.url,
- { method: HTTP_METHOD.HEAD, signal: options?.signal, ...identityEncodingConfig },
- { allowPrivateIP: options?.allowPrivateIP }
- )
-
- const headers = AxiosHeaders.from(headRes.headers)
- const contentLength = this.contentLength(headers)
- if (options?.getContentInfo) {
- return {
- contentLength,
- contentType: `${headers.getContentType()}`,
- lastModified: headers.get('last-modified') as string | undefined
- } satisfies DownloadFileContentInfo
+ // An explicit maximum allows chunked downloads; otherwise HEAD content-length is the stream boundary.
+ const streamMaxSize = options?.maxSize ?? contentLength
+ if (streamMaxSize === null || streamMaxSize === undefined) {
+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)
+ }
+ maxSize = streamMaxSize
+ const fileLimiter = createUploadStreamLimiter(options?.space, streamMaxSize).createFileLimiter()
+ if (contentLength !== null) {
+ fileLimiter.assertKnownSize(contentLength)
+ }
+ this.prepareSpaceTask(options?.space, contentLength, options?.publishedPath || dstPath)
+
+ // The HEAD request resolved redirects; the GET must target that final URL directly.
+ stage = 'get'
+ const { response: getRes } = await this.request(
+ currentUrl,
+ { method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },
+ { allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 }
+ )
+ stage = 'stream'
+ try {
+ const getContentLength = this.contentLength(AxiosHeaders.from(getRes.headers))
+ if (getContentLength !== null) {
+ fileLimiter.assertKnownSize(getContentLength)
+ }
+ } catch (e) {
+ getRes.data?.destroy?.()
+ throw e
+ }
+ await writeUploadFromStream(dstPath, getRes.data, {
+ limiter: fileLimiter,
+ signal: options?.signal,
+ onProgress: options?.onProgress
+ })
+ } catch (e) {
+ this.logFailure(stage, currentUrl, e, options?.signal, contentLength, maxSize ?? options?.maxSize)
+ throw e
}
+ }
- const maxSize = options?.space ? contentLength : (options?.maxSize ?? contentLength)
- if (maxSize === null || (contentLength === null && options?.space)) {
- throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)
- }
- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath)
-
- // The HEAD request resolved redirects; the GET must target that final URL directly.
- const { response: getRes } = await this.request(
- url,
- { method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },
- { allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 }
- )
- await writeFromStream(dstPath, getRes.data, 0, maxSize, options?.signal, options?.onProgress)
+ private logFailure(stage: DownloadStage, url: string, error: unknown, signal?: AbortSignal, contentLength?: number | null, maxSize?: number): void {
+ if (signal?.aborted) return
+ const host = URL.canParse(url) ? new URL(url).hostname || 'unknown' : 'invalid'
+ const sizeContext =
+ stage === 'validation' || stage === 'stream' ? ` (contentLength=${contentLength ?? 'missing'}, maxSize=${maxSize ?? 'unset'})` : ''
+ this.logger.warn({ tag: this.download.name, msg: `${stage} failed for host *${host}*${sizeContext}: ${error}` })
}
private safeLookup(hostname: string, options: Parameters<LookupFunction>[1], cb: Parameters<LookupFunction>[2]): void {
@@ -163,18 +207,29 @@ export class DownloadFile {
}
private contentLength(headers: AxiosHeaders): number | null {
- const value = headers.getContentLength(/^\d+$/)?.[0]
- const contentLength = Number(value)
- return Number.isSafeInteger(contentLength) ? contentLength : null
+ const value = headers.get('content-length')
+ if (value === undefined || value === null) return null
+
+ const normalized = typeof value === 'number' ? String(value) : typeof value === 'string' ? value.trim() : ''
+ if (!/^\d+$/.test(normalized)) {
+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)
+ }
+ const contentLength = Number(normalized)
+ if (!Number.isSafeInteger(contentLength)) {
+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)
+ }
+ return contentLength
}
- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string): void {
+ private prepareSpaceTask(space: SpaceEnv | undefined, contentLength: number | null, publishedPath: string): void {
if (!space) return
- if (space.willExceedQuota(contentLength)) {
- throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)
- }
if (space.task?.cacheKey) {
- space.task.props = { ...space.task.props, progress: 1, size: 0, totalSize: contentLength }
+ space.task.props = {
+ ...space.task.props,
+ progress: 1,
+ size: 0,
+ totalSize: contentLength ?? undefined
+ }
FileTaskEvent.emit('startWatch', space, publishedPath)
}
}
diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.ts
index 865de371..1ad5a33f 100644
--- a/backend/src/applications/files/utils/files.spec.ts
+++ b/backend/src/applications/files/utils/files.spec.ts
@@ -7,8 +7,21 @@ import fse from 'fs-extra'
import type { MockInstance } from 'vitest'
import { FileError } from '../models/file-error'
import { storageQuotaExceededError } from './errors'
-import { createSizeLimiter, isCrossDevice, isPathInside, makeTempDir, sanitizeName, tempFilePath, writeFromStream } from './files'
+import {
+ createSizeLimiter,
+ isCrossDevice,
+ isPathInside,
+ makeTempDir,
+ sanitizeName,
+ tempFilePath,
+ temporaryPathPrefix,
+ writeFromStream,
+ writeFromStreamAndChecksum,
+ writeUploadFromStream,
+ writeUploadFromStreamAndChecksum
+} from './files'
import { FILE_ERROR } from '../constants/errors'
+import { UploadStreamLimiter } from './upload-file'
describe(createSizeLimiter.name, () => {
it('rejects the call that makes the cumulative size exceed the limit', () => {
@@ -81,35 +94,118 @@ describe(writeFromStream.name, () => {
await rm(tmpDir, { recursive: true, force: true })
})
- it('writes from an offset up to the max size and reports progress', async () => {
+ it('writes from an offset and accounts each chunk', async () => {
+ const filePath = path.join(tmpDir, 'file.txt')
+ const accountBytes = vi.fn()
+ await writeFile(filePath, 'abc')
+
+ await writeFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), { start: 3, accountBytes })
+
+ await expect(readFile(filePath, 'utf8')).resolves.toBe('abcdef')
+ expect(accountBytes).toHaveBeenNthCalledWith(1, 2)
+ expect(accountBytes).toHaveBeenNthCalledWith(2, 1)
+ })
+
+ it('propagates a byte accounting error', async () => {
+ const filePath = path.join(tmpDir, 'file.txt')
+
+ await expect(
+ writeFromStream(filePath, Readable.from([Buffer.from('abcd')]), {
+ accountBytes: () => {
+ throw storageQuotaExceededError()
+ }
+ })
+ ).rejects.toMatchObject({
+ httpCode: HttpStatus.INSUFFICIENT_STORAGE,
+ message: FILE_ERROR.STORAGE_QUOTA_EXCEEDED,
+ name: FileError.name
+ })
+ })
+
+ it('aborts a stream when its signal is already aborted', async () => {
+ const filePath = path.join(tmpDir, 'file.txt')
+ const controller = new AbortController()
+ controller.abort()
+
+ await expect(writeFromStream(filePath, Readable.from([Buffer.from('abc')]), { signal: controller.signal })).rejects.toMatchObject({
+ name: 'AbortError'
+ })
+ })
+})
+
+describe(writeFromStreamAndChecksum.name, () => {
+ let tmpDir: string
+
+ beforeEach(async () => {
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'write-from-stream-checksum-'))
+ })
+
+ afterEach(async () => {
+ await rm(tmpDir, { recursive: true, force: true })
+ })
+
+ it('propagates a byte accounting error in checksum mode', async () => {
+ const filePath = path.join(tmpDir, 'file.txt')
+
+ await expect(
+ writeFromStreamAndChecksum(filePath, Readable.from([Buffer.from('abcd')]), 'sha256', {
+ accountBytes: () => {
+ throw storageQuotaExceededError()
+ }
+ })
+ ).rejects.toMatchObject({
+ httpCode: HttpStatus.INSUFFICIENT_STORAGE,
+ message: FILE_ERROR.STORAGE_QUOTA_EXCEEDED,
+ name: FileError.name
+ })
+ })
+})
+
+describe('upload stream writers', () => {
+ let tmpDir: string
+
+ beforeEach(async () => {
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'write-upload-from-stream-'))
+ })
+
+ afterEach(async () => {
+ await rm(tmpDir, { recursive: true, force: true })
+ })
+
+ it('uses the limiter offset and reports only accepted chunks', async () => {
const filePath = path.join(tmpDir, 'file.txt')
const onProgress = vi.fn()
+ const limiter = new UploadStreamLimiter(6).createFileLimiter(3)
await writeFile(filePath, 'abc')
- await writeFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), 3, 6, undefined, onProgress)
+ await writeUploadFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), { limiter, onProgress })
await expect(readFile(filePath, 'utf8')).resolves.toBe('abcdef')
expect(onProgress).toHaveBeenNthCalledWith(1, 2)
expect(onProgress).toHaveBeenNthCalledWith(2, 1)
})
- it('rejects a stream exceeding the max size', async () => {
+ it('does not report a chunk rejected by the limiter', async () => {
const filePath = path.join(tmpDir, 'file.txt')
+ const onProgress = vi.fn()
+ const limiter = new UploadStreamLimiter(3).createFileLimiter()
- await expect(writeFromStream(filePath, Readable.from([Buffer.from('abcd')]), 0, 3)).rejects.toMatchObject({
+ await expect(writeUploadFromStream(filePath, Readable.from([Buffer.from('abcd')]), { limiter, onProgress })).rejects.toMatchObject({
httpCode: HttpStatus.PAYLOAD_TOO_LARGE,
message: FILE_ERROR.MAX_FILE_SIZE_EXCEEDED,
name: FileError.name
})
+ expect(onProgress).not.toHaveBeenCalled()
})
- it('aborts a stream when its signal is already aborted', async () => {
+ it('enforces the limiter in checksum mode', async () => {
const filePath = path.join(tmpDir, 'file.txt')
- const controller = new AbortController()
- controller.abort()
+ const limiter = new UploadStreamLimiter(3).createFileLimiter()
- await expect(writeFromStream(filePath, Readable.from([Buffer.from('abc')]), 0, undefined, controller.signal)).rejects.toMatchObject({
- name: 'AbortError'
+ await expect(writeUploadFromStreamAndChecksum(filePath, Readable.from([Buffer.from('abcd')]), 'sha256', { limiter })).rejects.toMatchObject({
+ httpCode: HttpStatus.PAYLOAD_TOO_LARGE,
+ message: FILE_ERROR.MAX_FILE_SIZE_EXCEEDED,
+ name: FileError.name
})
})
})
@@ -148,3 +244,11 @@ describe(tempFilePath.name, () => {
expect(path.dirname(tempFilePath(parentPath, path.join('..', 'archive-')))).toBe(parentPath)
})
})
+
+describe(temporaryPathPrefix.name, () => {
+ it('uses the destination name and operation', () => {
+ expect(temporaryPathPrefix('/destination/report.pdf', 'upload')).toBe('report.pdf-upload-')
+ expect(temporaryPathPrefix('/destination', 'extract')).toBe('destination-extract-')
+ expect(temporaryPathPrefix('', 'download')).toBe('file-download-')
+ })
+})
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index 0c9714fc..f9b460f1 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -15,6 +15,7 @@ import type { FileDBProps } from '../interfaces/file-db-props.interface'
import type { FileProps } from '../interfaces/file-props.interface'
import { FileError } from '../models/file-error'
import { maxFileSizeExceededError } from './errors'
+import type { UploadFileStreamLimiter } from './upload-file'
export function sanitizePath(fPath: string): string {
return path.normalize(fPath).replace(regExpPreventPathTraversal, '')
@@ -118,6 +119,10 @@ export function tempFilePath(parentPath: string, prefix: string): string {
return path.join(parentPath, `${path.basename(prefix)}${crypto.randomUUID()}`)
}
+export function temporaryPathPrefix(targetPath: string, operation: 'upload' | 'download' | 'compress' | 'extract'): string {
+ return `${fileName(targetPath) || 'file'}-${operation}-`
+}
+
export function getMimeType(fPath: string, isDir: boolean): string {
if (isDir) {
return 'directory'
@@ -242,54 +247,90 @@ export function createProgressTransform(
})
}
-export function writeFromStream(
- rPath: string,
- stream: Readable,
- start: number = 0,
- maxSize?: number,
- signal?: AbortSignal,
- onProgress?: (bytes: number) => void
-): Promise<void> {
+export interface WriteFromStreamOptions {
+ start?: number
+ signal?: AbortSignal
+ // Called before forwarding a chunk to the destination. Throwing aborts the pipeline.
+ accountBytes?: (bytes: number) => void
+}
+
+export function writeFromStream(rPath: string, stream: Readable, options: WriteFromStreamOptions = {}): Promise<void> {
+ const { start = 0, signal, accountBytes } = options
const dst: WriteStream = createWriteStream(rPath, { flags: start ? 'a' : 'w', start: start, highWaterMark: DEFAULT_HIGH_WATER_MARK })
- if (maxSize === undefined && !onProgress) {
+ if (!accountBytes) {
return pipeline(stream, dst, { signal })
}
- let received = start
- const progress = new Transform({
+ const accounting = new Transform({
transform(chunk, _encoding, callback) {
- received += chunk.length
- if (maxSize !== undefined && received > maxSize) {
- callback(maxFileSizeExceededError())
+ try {
+ accountBytes(chunk.length)
+ } catch (error) {
+ callback(error as Error)
return
}
- onProgress?.(chunk.length)
callback(null, chunk)
}
})
- return pipeline(stream, progress, dst, { signal })
+ return pipeline(stream, accounting, dst, { signal })
}
-export async function writeFromStreamAndChecksum(rPath: string, stream: Readable, hasRange: number, alg: string): Promise<string> {
+export async function writeFromStreamAndChecksum(
+ rPath: string,
+ stream: Readable,
+ alg: string,
+ options: WriteFromStreamOptions = {}
+): Promise<string> {
+ const { start = 0, signal, accountBytes } = options
const hash = crypto.createHash(alg)
- if (hasRange) {
+ if (start) {
+ // Seed the hash with the existing prefix so the result covers the complete resumed file.
const src = createReadStream(rPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- await pipeline(src, hash, { end: false })
+ await pipeline(src, hash, { end: false, signal })
}
- const dst = createWriteStream(rPath, { flags: hasRange ? 'a' : 'w', highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ const dst = createWriteStream(rPath, { flags: start ? 'a' : 'w', highWaterMark: DEFAULT_HIGH_WATER_MARK })
await pipeline(
stream,
async function* (source) {
for await (const chunk of source) {
+ accountBytes?.(chunk.length)
hash.update(chunk)
yield chunk
}
},
- dst
+ dst,
+ { signal }
)
hash.end()
return hash.digest('hex')
}
+export interface WriteUploadStreamOptions {
+ limiter: UploadFileStreamLimiter
+ signal?: AbortSignal
+ // Receives only bytes accepted by the limiter.
+ onProgress?: (bytes: number) => void
+}
+
+function uploadWriteOptions({ limiter, signal, onProgress }: WriteUploadStreamOptions): WriteFromStreamOptions {
+ return {
+ start: limiter.initialFileSize,
+ signal,
+ accountBytes: (bytes) => {
+ // Progress must only include chunks accepted by both the file-size and quota boundaries.
+ limiter.consume(bytes)
+ onProgress?.(bytes)
+ }
+ }
+}
+
+export function writeUploadFromStream(rPath: string, stream: Readable, options: WriteUploadStreamOptions): Promise<void> {
+ return writeFromStream(rPath, stream, uploadWriteOptions(options))
+}
+
+export function writeUploadFromStreamAndChecksum(rPath: string, stream: Readable, alg: string, options: WriteUploadStreamOptions): Promise<string> {
+ return writeFromStreamAndChecksum(rPath, stream, alg, uploadWriteOptions(options))
+}
+
export function copyFileContent(srcPath: string, dstPath: string): Promise<void> {
const srcStream = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
return writeFromStream(dstPath, srcStream)
diff --git a/backend/src/applications/files/utils/upload-file.spec.ts b/backend/src/applications/files/utils/upload-file.spec.ts
new file mode 100644
index 00000000..f8dee0d2
--- /dev/null
+++ b/backend/src/applications/files/utils/upload-file.spec.ts
@@ -0,0 +1,79 @@
+import { HttpStatus } from '@nestjs/common'
+import { FILE_ERROR } from '../constants/errors'
+import { FileError } from '../models/file-error'
+import { parseContentLength, parseContentRange, UploadStreamLimiter } from './upload-file'
+
+describe(UploadStreamLimiter.name, () => {
+ it('short-circuits a known file size above maxUploadSize', () => {
+ const fileLimiter = new UploadStreamLimiter(10).createFileLimiter()
+
+ expect(() => fileLimiter.assertKnownSize(11)).toThrow(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))
+ })
+
+ it('short-circuits a known file size above the remaining quota', () => {
+ const fileLimiter = new UploadStreamLimiter(100, 4).createFileLimiter()
+
+ expect(() => fileLimiter.assertKnownSize(5)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
+ })
+
+ it('checks each streamed chunk before accepting it', () => {
+ const fileLimiter = new UploadStreamLimiter(5).createFileLimiter()
+
+ fileLimiter.consume(3)
+ expect(() => fileLimiter.consume(3)).toThrow(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))
+ })
+
+ it('shares quota consumption across multipart files', () => {
+ const limiter = new UploadStreamLimiter(100, 6)
+ limiter.createFileLimiter().consume(4)
+
+ const secondFile = limiter.createFileLimiter()
+ secondFile.consume(2)
+ expect(() => secondFile.consume(1)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
+ })
+
+ it('includes a range offset in maxUploadSize but not in newly consumed quota', () => {
+ const limiter = new UploadStreamLimiter(10, 5)
+ const rangedFile = limiter.createFileLimiter(8)
+
+ expect(rangedFile.initialFileSize).toBe(8)
+ rangedFile.assertFinalSize(10)
+ rangedFile.consume(2)
+ const secondFile = limiter.createFileLimiter()
+ expect(() => secondFile.consume(3)).not.toThrow()
+ expect(() => secondFile.consume(1)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))
+ })
+
+ it('rejects a final size below the current range offset', () => {
+ const rangedFile = new UploadStreamLimiter(10).createFileLimiter(8)
+
+ expect(() => rangedFile.assertFinalSize(7)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size'))
+ })
+})
+
+describe(parseContentLength.name, () => {
+ it('returns undefined when the header is absent', () => {
+ expect(parseContentLength(undefined)).toBeUndefined()
+ })
+
+ it('parses a valid byte count', () => {
+ expect(parseContentLength(' 42 ')).toBe(42)
+ })
+
+ it.each(['-1', '1.5', 'abc', '9007199254740992'])('rejects invalid content-length "%s"', (value) => {
+ expect(() => parseContentLength(value)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header'))
+ })
+})
+
+describe(parseContentRange.name, () => {
+ const malformedRanges = ['bytes 5-4/10', 'bytes 0-10/10', 'bytes nope', 'bytes 9007199254740992-9007199254740993/*']
+
+ it('parses ranges with a known or unknown total size', () => {
+ expect(parseContentRange('bytes 5-9/10')).toEqual({ start: 5, end: 9, total: 10 })
+ expect(parseContentRange('bytes 5-9/*')).toEqual({ start: 5, end: 9 })
+ })
+
+ it.each(malformedRanges)('rejects malformed range "%s"', (value) => {
+ expect(() => parseContentRange(value)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed'))
+ })
+})
diff --git a/backend/src/applications/files/utils/upload-file.ts b/backend/src/applications/files/utils/upload-file.ts
index 199a0eb0..daa1b577 100644
--- a/backend/src/applications/files/utils/upload-file.ts
+++ b/backend/src/applications/files/utils/upload-file.ts
@@ -1,14 +1,132 @@
+import { HttpStatus } from '@nestjs/common'
import path from 'node:path'
import { randomUUID } from 'node:crypto'
-import { fileName } from './files'
+import { FileError } from '../models/file-error'
+import { maxFileSizeExceededError, storageQuotaExceededError } from './errors'
+import { temporaryPathPrefix } from './files'
const FASTIFY_MULTIPART_FILE_TOO_LARGE_CODE = 'FST_REQ_FILE_TOO_LARGE' as const
+export interface UploadFileStreamLimiter {
+ readonly initialFileSize: number
+ // Checks the number of bytes the current stream is expected to append.
+ assertKnownSize: (bytes: number) => void
+ // Checks the expected final file size, including the current range offset.
+ assertFinalSize: (bytes: number) => void
+ consume: (bytes: number) => void
+}
+
+export interface UploadQuotaSnapshot {
+ storageQuota?: number | null
+ storageUsage?: number
+}
+
+/**
+ * Request-scoped upload limiter. It deliberately does not coordinate quota
+ * reservations between concurrent requests or workers.
+ */
+export class UploadStreamLimiter {
+ // Shared by all file limiters created for the same request, notably multipart uploads.
+ private quotaConsumed = 0
+
+ constructor(
+ private readonly maxFileSize: number,
+ private readonly quotaRemaining?: number
+ ) {}
+
+ createFileLimiter(initialFileSize = 0): UploadFileStreamLimiter {
+ if (!Number.isSafeInteger(initialFileSize) || initialFileSize < 0) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')
+ }
+ if (initialFileSize > this.maxFileSize) {
+ throw maxFileSizeExceededError()
+ }
+
+ // A resumed offset counts toward maxFileSize but is not charged again against the quota snapshot.
+ let fileSize = initialFileSize
+ const assertCanConsume = (bytes: number, consume: boolean) => {
+ if (!Number.isSafeInteger(bytes) || bytes < 0) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')
+ }
+
+ const fileRemaining = this.maxFileSize - fileSize
+ const quotaRemaining = this.quotaRemaining === undefined ? Number.POSITIVE_INFINITY : this.quotaRemaining - this.quotaConsumed
+
+ if (bytes > fileRemaining || bytes > quotaRemaining) {
+ // Report the first effective boundary. If both are equal, the file-size
+ // limit is the more specific upload error.
+ if (quotaRemaining < fileRemaining) {
+ throw storageQuotaExceededError()
+ }
+ throw maxFileSizeExceededError()
+ }
+
+ if (consume) {
+ fileSize += bytes
+ this.quotaConsumed += bytes
+ }
+ }
+
+ return {
+ initialFileSize,
+ assertKnownSize: (bytes) => assertCanConsume(bytes, false),
+ assertFinalSize: (bytes) => {
+ if (!Number.isSafeInteger(bytes) || bytes < fileSize) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')
+ }
+ // Convert the declared final size into the number of bytes still expected on this stream.
+ assertCanConsume(bytes - fileSize, false)
+ },
+ consume: (bytes) => assertCanConsume(bytes, true)
+ }
+ }
+}
+
+export function createUploadStreamLimiter(space: UploadQuotaSnapshot | undefined, maxFileSize: number): UploadStreamLimiter {
+ // A null quota means unlimited storage; a negative remainder is clamped so every positive write is rejected.
+ const quotaRemaining = space?.storageQuota == null ? undefined : Math.max(0, space.storageQuota - (space.storageUsage ?? 0))
+ return new UploadStreamLimiter(maxFileSize, quotaRemaining)
+}
+
+export function parseContentLength(value: unknown): number | undefined {
+ if (value === undefined || value === null) return undefined
+
+ const normalized = typeof value === 'number' ? String(value) : typeof value === 'string' ? value.trim() : ''
+ if (!/^\d+$/.test(normalized)) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header')
+ }
+
+ const contentLength = Number(normalized)
+ if (!Number.isSafeInteger(contentLength)) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header')
+ }
+ return contentLength
+}
+
+export function parseContentRange(value: string): { start: number; end: number; total?: number } {
+ const match = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(value)
+ if (!match) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')
+ }
+ const start = Number(match[1])
+ const end = Number(match[2])
+ const total = match[3] === '*' ? undefined : Number(match[3])
+ if (
+ !Number.isSafeInteger(start) ||
+ !Number.isSafeInteger(end) ||
+ end < start ||
+ (total !== undefined && (!Number.isSafeInteger(total) || total <= end))
+ ) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')
+ }
+ return { start, end, ...(total !== undefined && { total }) }
+}
+
export function isMultipartFileTooLargeError(e: any): boolean {
// Other multipart limits also return 413; only this code means the file-size limit was reached.
return e?.code === FASTIFY_MULTIPART_FILE_TOO_LARGE_CODE
}
export function uploadTmpFilePath(tmpPath: string, partFileName: string): string {
- return path.join(tmpPath, `${randomUUID()}-upload-${fileName(partFileName) || 'file'}`)
+ return path.join(tmpPath, `${temporaryPathPrefix(partFileName, 'upload')}${randomUUID()}`)
}
diff --git a/backend/src/applications/spaces/guards/space.guard.spec.ts b/backend/src/applications/spaces/guards/space.guard.spec.ts
index 1ead234d..7b4b305f 100644
--- a/backend/src/applications/spaces/guards/space.guard.spec.ts
+++ b/backend/src/applications/spaces/guards/space.guard.spec.ts
@@ -350,16 +350,17 @@ describe(SpaceGuard.name, () => {
await expect(spacesGuard.canActivate(context)).rejects.toEqual(expect.objectContaining({ status: HttpStatus.INSUFFICIENT_STORAGE }))
})
- it('should fail if request content-length header exceed the quota', async () => {
+ it('should leave generic content-length quota enforcement to the upload consumer', async () => {
userTest.role = USER_ROLE.USER
const storageQuota = 12
const storageUsage = 11
+ const willExceedQuota = vi.fn((contentLength: number) => contentLength > storageQuota - storageUsage)
spacesManager.spaceEnv = vi.fn().mockReturnValue({
enabled: true,
envPermissions: `${SPACE_OPERATION.ADD}`,
storageQuota: storageQuota,
storageUsage: storageUsage,
- willExceedQuota: (contentLength: number) => contentLength > storageQuota - storageUsage
+ willExceedQuota
} as Partial<SpaceEnv>)
context.switchToHttp().getRequest.mockReturnValueOnce({
method: 'POST',
@@ -367,14 +368,8 @@ describe(SpaceGuard.name, () => {
user: userTest,
params: { '*': 'files/personal' }
})
- await expect(spacesGuard.canActivate(context)).rejects.toEqual(expect.objectContaining({ status: HttpStatus.INSUFFICIENT_STORAGE }))
- context.switchToHttp().getRequest.mockReturnValueOnce({
- method: 'POST',
- user: userTest,
- headers: {},
- params: { '*': 'files/personal' }
- })
expect(await spacesGuard.canActivate(context)).toBe(true)
+ expect(willExceedQuota).not.toHaveBeenCalled()
})
it('should fail for add and modify operations in trash repository', async () => {
diff --git a/backend/src/applications/spaces/guards/space.guard.ts b/backend/src/applications/spaces/guards/space.guard.ts
index 8e4b50bd..27a9de5e 100644
--- a/backend/src/applications/spaces/guards/space.guard.ts
+++ b/backend/src/applications/spaces/guards/space.guard.ts
@@ -49,11 +49,6 @@ export class SpaceGuard implements CanActivate {
if (req.space.quotaIsExceeded) {
logger.warn(`Storage quota exceeded for *${req.space.alias}* (${req.space.id})`)
throw new HttpException(FILE_ERROR.STORAGE_QUOTA_EXCEEDED, HttpStatus.INSUFFICIENT_STORAGE)
- } else if (req.space.storageQuota) {
- const contentLength = parseInt(req.headers['content-length'] || '0', 10) || 0
- if (req.space.willExceedQuota(contentLength)) {
- throw new HttpException(FILE_ERROR.STORAGE_QUOTA_EXCEEDED, HttpStatus.INSUFFICIENT_STORAGE)
- }
}
}
}
diff --git a/backend/src/applications/sync/dtos/sync-upload.dto.ts b/backend/src/applications/sync/dtos/sync-upload.dto.ts
index f3e54fe4..9bdc2ce7 100644
--- a/backend/src/applications/sync/dtos/sync-upload.dto.ts
+++ b/backend/src/applications/sync/dtos/sync-upload.dto.ts
@@ -1,5 +1,5 @@
import { Transform } from 'class-transformer'
-import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from 'class-validator'
+import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString, Min } from 'class-validator'
export class SyncUploadDto {
@IsOptional()
@@ -17,5 +17,6 @@ export class SyncUploadDto {
@IsNotEmpty()
@Transform(({ value }) => parseInt(value))
@IsInt()
+ @Min(0)
size: number
}
diff --git a/backend/src/applications/sync/services/sync-manager.service.spec.ts b/backend/src/applications/sync/services/sync-manager.service.spec.ts
index ac0ad836..2e3db7a2 100644
--- a/backend/src/applications/sync/services/sync-manager.service.spec.ts
+++ b/backend/src/applications/sync/services/sync-manager.service.spec.ts
@@ -165,6 +165,7 @@ describe(SyncManager.name, () => {
const r = await service.upload(req as any, dto as any)
expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {
tmpPath: expect.any(String),
+ expectedUploadSize: 10,
checksumAlg: SYNC_CHECKSUM_ALG,
validateTmpFile: expect.any(Function)
})
@@ -195,6 +196,7 @@ describe(SyncManager.name, () => {
const r = await service.upload(req as any, dto as any)
expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {
tmpPath: expect.any(String),
+ expectedUploadSize: 5,
validateTmpFile: expect.any(Function)
})
expect(touchFile).toHaveBeenCalledWith('/tmp/up2.bin', 1710000100)
@@ -214,9 +216,10 @@ describe(SyncManager.name, () => {
expect(touchFile).not.toHaveBeenCalled()
})
- it('should reject when tmp file exceeds quota and preserve tmp', async () => {
+ it('should delegate the declared size to the streaming limiter without a post-write quota check', async () => {
+ const willExceedQuota = vi.fn().mockReturnValue(true)
const req = makeReq({
- space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota: vi.fn().mockReturnValue(true) }
+ space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota }
})
const dto = { size: 10, mtime: 1710000100 }
filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {
@@ -224,10 +227,11 @@ describe(SyncManager.name, () => {
})
fsPromises.stat.mockResolvedValue({ size: 10, ino: 321, mtime: new Date(1710000100 * 1000) })
- await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)
- expect(req.space.willExceedQuota).toHaveBeenCalledWith(10)
+ await expect(service.upload(req as any, dto as any)).resolves.toEqual({ ino: 321 })
+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, expect.objectContaining({ expectedUploadSize: 10 }))
+ expect(willExceedQuota).not.toHaveBeenCalled()
expect(removeFiles).not.toHaveBeenCalled()
- expect(touchFile).not.toHaveBeenCalled()
+ expect(touchFile).toHaveBeenCalled()
})
})
@@ -585,7 +589,7 @@ describe(SyncManager.name, () => {
await expect(
(async () => {
for await (const _ of iter) {
- /* consume */
+ // consume
}
})()
).rejects.toThrow('Unable to parse path')
diff --git a/backend/src/applications/sync/services/sync-manager.service.ts b/backend/src/applications/sync/services/sync-manager.service.ts
index aa672ee5..a2d5cd2b 100644
--- a/backend/src/applications/sync/services/sync-manager.service.ts
+++ b/backend/src/applications/sync/services/sync-manager.service.ts
@@ -50,15 +50,13 @@ export class SyncManager {
try {
await this.filesManager.saveStream(req.user, req.space, req, {
tmpPath: tmpPath,
+ expectedUploadSize: syncUploadDto.size,
...(syncUploadDto.checksum && { checksumAlg: SYNC_CHECKSUM_ALG }),
validateTmpFile: async ({ tmpPath, checksum }) => {
const tmpStats = await fs.stat(tmpPath)
if (tmpStats.size !== syncUploadDto.size) {
throw new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${tmpStats.size} != ${syncUploadDto.size}`)
}
- if (req.space.storageQuota && req.space.willExceedQuota(tmpStats.size)) {
- throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)
- }
if (syncUploadDto.checksum && checksum !== syncUploadDto.checksum) {
throw new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical')
}
diff --git a/backend/src/applications/webdav/services/webdav-methods.service.spec.ts b/backend/src/applications/webdav/services/webdav-methods.service.spec.ts
index ecd71f84..20a5d4af 100644
--- a/backend/src/applications/webdav/services/webdav-methods.service.spec.ts
+++ b/backend/src/applications/webdav/services/webdav-methods.service.spec.ts
@@ -6,7 +6,7 @@ import { FileError } from '../../files/models/file-error'
import { LockConflict } from '../../files/models/file-lock-error'
import { FilesLockManager } from '../../files/services/files-lock-manager.service'
import { FilesManager } from '../../files/services/files-manager.service'
-import { dirName, genEtag, isPathExists } from '../../files/utils/files'
+import { dirName, genEtag, isPathExists, removeFiles } from '../../files/utils/files'
import { SPACE_REPOSITORY } from '../../spaces/constants/spaces'
import * as PathsUtils from '../../spaces/utils/paths'
import { haveSpaceEnvPermissions } from '../../spaces/utils/permissions'
@@ -22,7 +22,10 @@ vi.mock('../../files/utils/files', () => ({
isPathIsDir: vi.fn(),
fileName: vi.fn().mockReturnValue('fileName'),
dirName: vi.fn(),
- genEtag: vi.fn().mockReturnValue('W/"etag-123"')
+ genEtag: vi.fn().mockReturnValue('W/"etag-123"'),
+ removeFiles: vi.fn(() => Promise.resolve()),
+ tempFilePath: vi.fn().mockReturnValue('/tmp/test-user/file.txt-upload-id'),
+ temporaryPathPrefix: vi.fn().mockReturnValue('file.txt-upload-')
}))
vi.mock('../../spaces/utils/permissions', () => ({
@@ -75,7 +78,8 @@ describe('WebDAVMethods', () => {
const createBaseRequest = (overrides: Partial<any> = {}) =>
({
method: 'GET',
- user: { id: 1, login: 'test-user', fullName: 'Test User', email: '[email protected]' },
+ headers: {},
+ user: { id: 1, login: 'test-user', fullName: 'Test User', email: '[email protected]', tmpPath: '/tmp/test-user' },
dav: {
url: '/webdav/test/file.txt',
depth: '0',
@@ -631,7 +635,12 @@ describe('WebDAVMethods', () => {
const result = await service.put(req, res)
- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, expect.objectContaining({ dav: expect.any(Object) }))
+ expect(filesManager.saveStream).toHaveBeenCalledWith(
+ req.user,
+ req.space,
+ req,
+ expect.objectContaining({ dav: expect.any(Object), tmpPath: '/tmp/test-user/file.txt-upload-id' })
+ )
expect(res.statusCode).toBe(HttpStatus.NO_CONTENT)
expect(res.headers['etag']).toBeDefined()
expect(result).toBe(res)
@@ -641,9 +650,11 @@ describe('WebDAVMethods', () => {
filesManager.saveStream.mockResolvedValue(false) // file didn't exist
const req = createBaseRequest({ method: 'PUT' })
const res = createMockResponse()
+ vi.mocked(isPathExists).mockImplementation(async (rPath: string) => rPath === '/real/path/to')
const result = await service.put(req, res)
+ expect(filesManager.saveStream.mock.calls[0][3]).not.toHaveProperty('tmpPath')
expect(res.statusCode).toBe(HttpStatus.CREATED)
expect(res.headers['etag']).toBeDefined()
expect(result).toBe(res)
@@ -674,6 +685,16 @@ describe('WebDAVMethods', () => {
})
)
})
+
+ it('should write content-range updates directly to preserve resumability', async () => {
+ filesManager.saveStream.mockResolvedValue(true)
+ const req = createBaseRequest({ method: 'PUT', headers: { 'content-range': 'bytes 4-7/8' } })
+ const res = createMockResponse()
+
+ await service.put(req, res)
+
+ expect(filesManager.saveStream.mock.calls[0][3]).not.toHaveProperty('tmpPath')
+ })
})
describe('Error handling', () => {
@@ -704,6 +725,17 @@ describe('WebDAVMethods', () => {
expect(res.statusCode).toBe(409)
expect(res.body).toBe('File conflict')
+ expect(removeFiles).toHaveBeenCalledWith('/tmp/test-user/file.txt-upload-id')
+ })
+
+ it('should not clean up direct content-range updates after an error', async () => {
+ filesManager.saveStream.mockRejectedValue(new FileError(HttpStatus.BAD_REQUEST, 'Interrupted upload'))
+ const req = createBaseRequest({ method: 'PUT', headers: { 'content-range': 'bytes 4-7/8' } })
+ const res = createMockResponse()
+
+ await service.put(req, res)
+
+ expect(removeFiles).not.toHaveBeenCalled()
})
it('should throw HttpException for unexpected errors', async () => {
@@ -1524,7 +1556,7 @@ describe('WebDAVMethods', () => {
describe('haveLock condition', () => {
it('should return true when haveLock matches (lock exists, mustMatch=true)', async () => {
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockResolvedValue([{ key: 'lock1' }] as any)
const req = createBaseRequest({
dav: {
@@ -1540,7 +1572,7 @@ describe('WebDAVMethods', () => {
})
it('should return true when haveLock matches (no lock, mustMatch=false)', async () => {
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockResolvedValue([])
const req = createBaseRequest({
dav: {
@@ -1556,7 +1588,7 @@ describe('WebDAVMethods', () => {
})
it('should return false with 412 when haveLock mismatches (lock exists, mustMatch=false)', async () => {
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockResolvedValue([{ key: 'lock1' }] as any)
const req = createBaseRequest({
dav: {
@@ -1573,7 +1605,7 @@ describe('WebDAVMethods', () => {
})
it('should return false with 412 when haveLock mismatches (no lock, mustMatch=true)', async () => {
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockResolvedValue([])
const req = createBaseRequest({
dav: {
@@ -1590,7 +1622,7 @@ describe('WebDAVMethods', () => {
})
it('should return false with 412 when haveLock lookup throws error', async () => {
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockRejectedValue(new Error('Database error'))
const req = createBaseRequest({
dav: {
@@ -1797,7 +1829,7 @@ describe('WebDAVMethods', () => {
describe('Multiple conditions', () => {
it('should evaluate multiple conditions and return true if any matches', async () => {
vi.mocked(isPathExists).mockResolvedValue(true)
- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })
+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })
filesLockManager.getLocksByPath.mockResolvedValue([])
const req = createBaseRequest({
dav: {
diff --git a/backend/src/applications/webdav/services/webdav-methods.service.ts b/backend/src/applications/webdav/services/webdav-methods.service.ts
index a8e8ae21..35982d1c 100644
--- a/backend/src/applications/webdav/services/webdav-methods.service.ts
+++ b/backend/src/applications/webdav/services/webdav-methods.service.ts
@@ -7,7 +7,7 @@ import { FileError } from '../../files/models/file-error'
import { LockConflict } from '../../files/models/file-lock-error'
import { FilesLockManager } from '../../files/services/files-lock-manager.service'
import { FilesManager } from '../../files/services/files-manager.service'
-import { dirName, fileName, genEtag, isPathExists, isPathIsDir } from '../../files/utils/files'
+import { dirName, fileName, genEtag, isPathExists, isPathIsDir, removeFiles, tempFilePath, temporaryPathPrefix } from '../../files/utils/files'
import { SPACE_OPERATION, SPACE_REPOSITORY } from '../../spaces/constants/spaces'
import { SpaceEnv } from '../../spaces/models/space-env.model'
import { dbFileFromSpace } from '../../spaces/utils/paths'
@@ -216,15 +216,33 @@ export class WebDAVMethods {
@IfHeaderDecorator()
async put(req: FastifyDAVRequest, res: FastifyReply): Promise<FastifyReply> {
+ const isRangeUpload = req.headers['content-range'] !== undefined
+ let tmpPath: string | undefined
let rExists: boolean
try {
+ // saveStream creates the parent of a supplied tmpPath, so keep the WebDAV destination
+ // parent check here to preserve the expected 409 response.
+ if (!(await isPathExists(dirName(req.space.realPath)))) {
+ throw new FileError(HttpStatus.CONFLICT, 'Parent must exists')
+ }
+ // Stage only complete overwrites to preserve the previous file on failure. New files and
+ // ranged uploads write directly because their partial content is the checkpoint used to resume.
+ if (!isRangeUpload && (await isPathExists(req.space.realPath))) {
+ tmpPath = tempFilePath(req.user.tmpPath, temporaryPathPrefix(req.space.realPath, 'upload'))
+ }
rExists = await this.filesManager.saveStream(req.user, req.space, req, {
+ ...(tmpPath && { tmpPath }),
dav: {
depth: req.dav.depth,
lockTokens: extractAllTokens(req.dav.ifHeaders)
}
})
} catch (e) {
+ if (tmpPath) {
+ await removeFiles(tmpPath).catch((cleanupError: Error) =>
+ this.logger.error({ tag: this.put.name, msg: `Unable to remove WebDAV staging file ${tmpPath}: ${cleanupError}` })
+ )
+ }
return this.handleError(req, res, e)
}
return res
diff --git a/backend/src/applications/webdav/utils/bootstrap.ts b/backend/src/applications/webdav/utils/bootstrap.ts
index c9e1e2a6..ed0818fa 100644
--- a/backend/src/applications/webdav/utils/bootstrap.ts
+++ b/backend/src/applications/webdav/utils/bootstrap.ts
@@ -29,6 +29,8 @@ export function bootstrapWebDAV(app: NestFastifyApplication, fastifyInstance: Fa
* WebDAV clients may send JSON/XML files with a Content-Type that triggers
* Fastify's body parsers (application/json, text/plain, application/xml...).
* For PUT uploads we must always treat the payload as a raw binary stream.
+ * Sync uploads are intentionally not handled here: the Sync client contract
+ * requires application/octet-stream, while other Sync POST routes use parsed bodies.
*
* This hook forces `application/octet-stream` for WebDAV PUT requests only,
* leaving XML-based WebDAV methods unaffected.
diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yaml
index f1156719..1b28f7e1 100755
--- a/environment/environment.dist.yaml
+++ b/environment/environment.dist.yaml
@@ -331,7 +331,7 @@ applications:
files:
# required
dataPath: /home/sync-in
- # maxUploadSize: Maximum upload file size.
+ # maxUploadSize: Maximum size in bytes for a file received through uploads, WebDAV, sync, remote downloads, or editor callbacks.
# default: 5368709120 (5 GB)
maxUploadSize: 5368709120
contentIndexing: