Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): enforce quotas while streaming uploads
johaven8107d40750da
diff --git a/backend/src/app.bootstrap.ts b/backend/src/app.bootstrap.tsindex b612e044..204e9ec2 100644--- a/backend/src/app.bootstrap.ts+++ b/backend/src/app.bootstrap.ts@@ -22,13 +22,13 @@ export async function appBootstrap(): Promise<NestFastifyApplication> {ignoreTrailingSlash: true,maxParamLength: 256},- bodyLimit: 26214400 /* 25 MB */+ bodyLimit: 26214400 // 25 MB})const app: NestFastifyApplication = await NestFactory.create<NestFastifyApplication>(AppModule, fastifyAdapter, {bufferLogs: true})- /* NestJS starts listening for shutdown hooks */+ // NestJS starts listening for shutdown hooksapp.enableShutdownHooks()/* Fastify instance */@@ -41,8 +41,9 @@ export async function appBootstrap(): Promise<NestFastifyApplication> {bootstrapWebDAV(app, fastifyInstance)/* PARSER */- // '*' body parser allow binary data as a stream (unlimited body size)- fastifyInstance.addContentTypeParser('*', { bodyLimit: 0 }, (_req: FastifyRequest, _payload: FastifyRequest['raw'], done) => done(null))+ // Keep unknown binary payloads available through req.raw.+ // Fastify's bodyLimit still protects buffered parsers; upload consumers meter this raw stream.+ fastifyInstance.addContentTypeParser('*', (_req: FastifyRequest, _payload: FastifyRequest['raw'], done) => done(null))/* INTERCEPTORS */app.useGlobalInterceptors(diff --git a/backend/src/app.service.spec.ts b/backend/src/app.service.spec.tsindex 4f63f48b..54abb867 100644--- a/backend/src/app.service.spec.ts+++ b/backend/src/app.service.spec.ts@@ -83,6 +83,7 @@ describe(AppService.name, () => {expect(conf.logger.colorize).toBe(true)const tmpSecretFile = path.join(os.tmpdir(), 'secret')fs.writeFileSync(tmpSecretFile, 'fooBAR8888')+ vi.spyOn(console, 'log').mockImplementation(() => undefined)const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)process.env[`${ENVIRONMENT_PREFIX}APPLICATIONS_FILES_ONLYOFFICE_SECRET`] = 'fooBAR'process.env[`${ENVIRONMENT_PREFIX}LOGGER_STDOUT`] = 'false'diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.tsindex 92b892e3..f4c00860 100644--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.spec.ts@@ -14,6 +14,7 @@ import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'import { FILE_MODE } from '../../constants/operations'import { FileEvent } from '../../events/file-events'import { FileLockProps } from '../../interfaces/file-props.interface'+import { FileError } from '../../models/file-error'import { LockConflict } from '../../models/file-lock-error'import { FilesLockManager } from '../../services/files-lock-manager.service'import * as filesUtils from '../../utils/files'@@ -319,7 +320,7 @@ describe(CollaboraOnlineManager.name, () => {vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)vi.spyOn(filesUtils, 'fileName').mockReturnValue('document.docx')vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/document-unique.docx')- vi.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)+ vi.spyOn(filesUtils, 'writeUploadFromStream').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'copyFileContent').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)})@@ -347,7 +348,11 @@ describe(CollaboraOnlineManager.name, () => {expect(result).toEqual({LastModifiedTime: '2024-01-01T11:00:00.000Z'})- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/document-unique.docx', mockRequest.raw)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/document-unique.docx',+ mockRequest.raw,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesUtils.copyFileContent).toHaveBeenCalledWith('/tmp/document-unique.docx', mockSpace.realPath)expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')expect(emitSpy).toHaveBeenCalledWith('event', {@@ -374,6 +379,36 @@ describe(CollaboraOnlineManager.name, () => {vi.spyOn(fs, 'stat').mockResolvedValue({ mtime: new Date('2024-01-01T10:00:00.000Z') } as any)await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST))+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')+ })++ it('should reject a known document size above the remaining quota before writing', async () => {+ const mockRequest = {+ user: mockUser,+ space: { ...mockSpace, storageQuota: 10, storageUsage: 9 },+ headers: { 'content-length': '2' },+ raw: new Readable()+ } as unknown as FastifyCollaboraOnlineSpaceRequest++ await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('Storage quota exceeded', HttpStatus.INSUFFICIENT_STORAGE))++ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')+ })++ it('should preserve the upload size error and remove the temporary file', async () => {+ const mockRequest = {+ user: mockUser,+ space: mockSpace,+ headers: {},+ raw: new Readable()+ } as unknown as FastifyCollaboraOnlineSpaceRequest++ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, 'File size limit exceeded'))++ await expect(service.saveDocument(mockRequest)).rejects.toThrow(new HttpException('File size limit exceeded', HttpStatus.PAYLOAD_TOO_LARGE))+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/document-unique.docx')+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()})it('should throw error when timestamp mismatch', async () => {diff --git a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.tsindex ae35d09c..e40e2ba4 100644--- a/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts+++ b/backend/src/applications/files/editors/collabora-online/collabora-online-manager.service.ts@@ -16,6 +16,7 @@ import { DEPTH, LOCK_SCOPE } from '../../../webdav/constants/webdav'import { FILE_MODE } from '../../constants/operations'import { FileLockOptions } from '../../interfaces/file-lock.interface'import { FileLockProps } from '../../interfaces/file-props.interface'+import { FileError } from '../../models/file-error'import { LockConflict } from '../../models/file-lock-error'import { FilesLockManager } from '../../services/files-lock-manager.service'import {@@ -28,7 +29,7 @@ import {isPathIsDir,removeFiles,uniqueFilePathFromDir,- writeFromStream+ writeUploadFromStream} from '../../utils/files'import {COLLABORA_APP_LOCK,@@ -44,10 +45,12 @@ import type { CollaboraOnlineCheckFileInfo, FastifyCollaboraOnlineSpaceRequest,import { API_COLLABORA_ONLINE_FILES } from './collabora-online.routes'import { FileEvent } from '../../events/file-events'import { ACTION } from '../../../../common/constants'+import { createUploadStreamLimiter, parseContentLength } from '../../utils/upload-file'@Injectable()export class CollaboraOnlineManager {private logger = new Logger(CollaboraOnlineManager.name)+ private readonly maxUploadSize = configuration.applications.files.maxUploadSizeprivate readonly externalCollaboraOnlineServer = configuration.applications.files.editors.collabora.externalServer || nullprivate readonly expiration = convertHumanTimeToSeconds(configuration.auth.token.refresh.expiration)@@ -116,33 +119,48 @@ export class CollaboraOnlineManager {await this.checkSpace(req.space)await this.checkTimeStampFromHeaders(req)const tmpFilePath = await uniqueFilePathFromDir(path.join(os.tmpdir(), fileName(req.space.realPath)))+ let contentLength: number | undefinedtry {- await writeFromStream(tmpFilePath, req.raw)- } catch (e) {- throw new Error(`unable to save document : ${e.message}`)- }- // try to verify the downloaded size- const contentLength = parseInt(req.headers['content-length'], 10)- if (!isNaN(contentLength) && contentLength !== 0) {- const tmpFileSize = await fileSize(tmpFilePath)- if (tmpFileSize !== contentLength) {- this.logger.error({ tag: this.saveDocument.name, msg: `document size differs (${tmpFileSize} != ${contentLength})` })- throw new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST)+ try {+ const fileLimiter = createUploadStreamLimiter(req.space, this.maxUploadSize).createFileLimiter()+ contentLength = parseContentLength(req.headers['content-length'])+ if (contentLength !== undefined) {+ fileLimiter.assertKnownSize(contentLength)+ }+ await writeUploadFromStream(tmpFilePath, req.raw, { limiter: fileLimiter })+ } catch (e) {+ if (e instanceof FileError) {+ throw new HttpException(e.message, e.httpCode)+ }+ throw new Error(`unable to save document : ${e.message}`)+ }+ // try to verify the downloaded size+ if (contentLength !== undefined && contentLength !== 0) {+ const tmpFileSize = await fileSize(tmpFilePath)+ if (tmpFileSize !== contentLength) {+ this.logger.error({ tag: this.saveDocument.name, msg: `document size differs (${tmpFileSize} != ${contentLength})` })+ throw new HttpException('Size Mismatch', HttpStatus.BAD_REQUEST)+ }+ } else if (contentLength === 0) {+ this.logger.warn({ tag: this.saveDocument.name, msg: `content length is 0 : ${req.space.url}` })+ }+ // copy contents to avoid inode changes (dbFileHash in some cases)+ try {+ await copyFileContent(tmpFilePath, req.space.realPath)+ // emit file event+ FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })+ const fStats = await fs.stat(req.space.realPath)+ return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto+ } catch (e) {+ this.logger.error({ tag: this.saveDocument.name, msg: `unable to save document: ${e}` })+ throw new HttpException('Unable to save document', HttpStatus.INTERNAL_SERVER_ERROR)+ }+ } finally {+ try {+ await removeFiles(tmpFilePath)+ } catch (e) {+ this.logger.error({ tag: this.saveDocument.name, msg: `failed to remove temporary document ${tmpFilePath}: ${e.message}` })}- } else if (contentLength === 0) {- this.logger.warn({ tag: this.saveDocument.name, msg: `content length is 0 : ${req.space.url}` })- }- // copy contents to avoid inode changes (dbFileHash in some cases)- try {- await copyFileContent(tmpFilePath, req.space.realPath)- // emit file event- FileEvent.emit('event', { user: req.user, space: req.space, action: ACTION.UPDATE, rPath: req.space.realPath, source: 'editor' })- await removeFiles(tmpFilePath)- const fStats = await fs.stat(req.space.realPath)- return { LastModifiedTime: fStats.mtime.toISOString() } satisfies CollaboraSaveDocumentDto- } catch (e) {- this.logger.error({ tag: this.saveDocument.name, msg: `unable to save document: ${e}` })- throw new HttpException('Unable to save document', HttpStatus.INTERNAL_SERVER_ERROR)}}diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.tsindex c4101010..8d97fe45 100644--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.spec.ts@@ -17,6 +17,7 @@ import { FILE_MODE } from '../../constants/operations'import { FileEvent } from '../../events/file-events'import { LockConflict } from '../../models/file-lock-error'import { FilesLockManager } from '../../services/files-lock-manager.service'+import { maxFileSizeExceededError } from '../../utils/errors'import * as filesUtils from '../../utils/files'import { OnlyOfficeManager } from './only-office-manager.service'import { ONLY_OFFICE_APP_LOCK } from './only-office.constants'@@ -260,7 +261,7 @@ describe(OnlyOfficeManager.name, () => {filesLockManager.isPathLocked.mockResolvedValue(false)cache.del.mockResolvedValue(true)vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValue('/tmp/temp-file.docx')- vi.mocked(filesUtils.writeFromStream).mockResolvedValue(undefined)+ vi.mocked(filesUtils.writeUploadFromStream).mockResolvedValue(undefined)vi.mocked(filesUtils.fileSize).mockResolvedValue(12)vi.mocked(filesUtils.copyFileContent).mockResolvedValue(undefined)vi.mocked(filesUtils.removeFiles).mockResolvedValue(undefined)@@ -299,6 +300,11 @@ describe(OnlyOfficeManager.name, () => {source: 'editor'})expect(httpService.axiosRef).toHaveBeenCalledWith(expect.objectContaining({ url, maxRedirects: 0 }))+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ expect.stringContaining('sync-in-onlyoffice-'),+ expect.anything(),+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )}it('should allow internal container document downloads when external server is not configured', async () => {@@ -404,6 +410,41 @@ describe(OnlyOfficeManager.name, () => {expect(result.error).not.toBe(0)})+ it('should reject an oversized editor document and remove the temporary file', async () => {+ jwtService.verifyAsync.mockResolvedValue({+ status: 2,+ actions: [],+ notmodified: false,+ url: mockDocumentUrl+ })+ mockSuccessfulDownload()+ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(maxFileSizeExceededError())++ const result = await service.callBack(mockUser, mockSpaceEnv, mockToken)++ expect(result).toEqual({ error: 'unable to download document : File size limit exceeded' })+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(expect.stringContaining('sync-in-onlyoffice-'))+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()+ })++ it('should reject an editor document above the remaining quota before writing', async () => {+ const quotaSpace = { ...mockSpaceEnv, storageQuota: 12, storageUsage: 1 } as unknown as SpaceEnv+ jwtService.verifyAsync.mockResolvedValue({+ status: 2,+ actions: [],+ notmodified: false,+ url: mockDocumentUrl+ })+ mockSuccessfulDownload()++ const result = await service.callBack(mockUser, quotaSpace, mockToken)++ expect(result).toEqual({ error: 'unable to download document : Storage quota exceeded' })+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(expect.stringContaining('sync-in-onlyoffice-'))+ expect(filesUtils.copyFileContent).not.toHaveBeenCalled()+ })+it('should reject document downloads from unexpected hosts when external server is configured', async () => {;(service as any).externalOnlyOfficeServer = 'http://localhost:3000/onlyoffice'jwtService.verifyAsync.mockResolvedValue({diff --git a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts b/backend/src/applications/files/editors/only-office/only-office-manager.service.tsindex 072408e5..c2911144 100644--- a/backend/src/applications/files/editors/only-office/only-office-manager.service.ts+++ b/backend/src/applications/files/editors/only-office/only-office-manager.service.ts@@ -34,7 +34,7 @@ import {isPathExists,isPathIsDir,removeFiles,- writeFromStream+ writeUploadFromStream} from '../../utils/files'import {EURO_OFFICE_APP_LOCK,@@ -51,10 +51,12 @@ import { OnlyOfficeCallBack, OnlyOfficeConfig, OnlyOfficeConvertForm } from './oimport { API_ONLY_OFFICE_CALLBACK, API_ONLY_OFFICE_DOCUMENT } from './only-office.routes'import { FileEvent } from '../../events/file-events'import { ACTION } from '../../../../common/constants'+import { createUploadStreamLimiter, parseContentLength } from '../../utils/upload-file'@Injectable()export class OnlyOfficeManager {private logger = new Logger(OnlyOfficeManager.name)+ private readonly maxUploadSize = configuration.applications.files.maxUploadSizeprivate readonly officeConfig = configuration.applications.files.editors.onlyoffice.enabled? configuration.applications.files.editors.onlyoffice: configuration.applications.files.editors.eurooffice@@ -350,17 +352,25 @@ export class OnlyOfficeManager {}// download file- let res: AxiosResponse+ let downloadStream: AxiosResponse['data'] | undefined+ let contentLength: number | undefinedtry {- res = await this.http.axiosRef({+ const res: AxiosResponse = await this.http.axiosRef({method: HTTP_METHOD.GET,url: finalDownloadUrl,responseType: 'stream',maxRedirects: 0,httpsAgent: new https.Agent({ rejectUnauthorized: this.rejectUnauthorized })})- await writeFromStream(tmpFilePath, res.data)+ downloadStream = res.data+ const fileLimiter = createUploadStreamLimiter(space, this.maxUploadSize).createFileLimiter()+ contentLength = parseContentLength(res.headers['content-length'])+ if (contentLength !== undefined) {+ fileLimiter.assertKnownSize(contentLength)+ }+ await writeUploadFromStream(tmpFilePath, downloadStream, { limiter: fileLimiter })} catch (e) {+ downloadStream?.destroy?.()this.logger.error({tag: this.saveDocument.name,msg: `failed to download document from ${finalParsedDownloadUrl.origin}${finalParsedDownloadUrl.pathname} : ${space.url} - ${e.message}`@@ -369,8 +379,7 @@ export class OnlyOfficeManager {}// try to verify the downloaded size- const contentLength = Number(res.headers['content-length'])- if (!isNaN(contentLength) && contentLength !== 0) {+ if (contentLength !== undefined && contentLength !== 0) {const tmpFileSize = await fileSize(tmpFilePath)if (tmpFileSize !== contentLength) {throw new Error(`document size differs (${tmpFileSize} != ${contentLength})`)diff --git a/backend/src/applications/files/interfaces/save-stream.interface.ts b/backend/src/applications/files/interfaces/save-stream.interface.tsindex d2deb6ba..e2c81527 100644--- a/backend/src/applications/files/interfaces/save-stream.interface.ts+++ b/backend/src/applications/files/interfaces/save-stream.interface.ts@@ -9,6 +9,9 @@ export interface SaveStreamTmpFileValidationContext {export interface SaveStreamOptions {dav?: { depth: LOCK_DEPTH; lockTokens: string[] }checksumAlg?: string+ // Expected final file size, including any resumed range.+ expectedUploadSize?: number+ maxSize?: numbertmpPath?: stringvalidateTmpFile?: (ctx: SaveStreamTmpFileValidationContext) => Promise<void>}diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 4a3cfb79..0fa61536 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -6,6 +6,7 @@ import path from 'node:path'import { Readable } from 'node:stream'import { transformAndValidate } from '../../../common/functions'import * as imageUtils from '../../../common/image'+import { configuration } from '../../../configuration/config.environment'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'import { NotificationsManager } from '../../notifications/services/notifications-manager.service'import { SpacesManager } from '../../spaces/services/spaces-manager.service'@@ -95,8 +96,8 @@ describe(FilesManager.name, () => {})const expectNoWriteOperations = () => {- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()- expect(filesUtils.writeFromStreamAndChecksum).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStreamAndChecksum).not.toHaveBeenCalled()expect(filesUtils.makeDir).not.toHaveBeenCalled()expect(filesUtils.createEmptyFile).not.toHaveBeenCalled()expect(filesUtils.copyFileContent).not.toHaveBeenCalled()@@ -185,8 +186,8 @@ describe(FilesManager.name, () => {vi.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)vi.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')vi.spyOn(filesUtils, 'tempFilePath').mockReturnValue('/tmp/staged-file')- vi.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)- vi.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')+ vi.spyOn(filesUtils, 'writeUploadFromStream').mockResolvedValue(undefined)+ vi.spyOn(filesUtils, 'writeUploadFromStreamAndChecksum').mockResolvedValue('sha256-abc')vi.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'copyFiles').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)@@ -243,11 +244,55 @@ describe(FilesManager.name, () => {expect(result).toBe(false)expect(filesLockManager.create).toHaveBeenCalledWith(user, space.dbFile, 'Sync-in', DEPTH.RESOURCE)- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(space.realPath, expect.anything(), 0)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ space.realPath,+ expect.anything(),+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })+ )expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: space.realPath })})+ it('should short-circuit a known body above a stricter max size supplied by the caller', async () => {+ const space = makeSpace()+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)++ await expect(+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-length': '5' }, raw: Readable.from(['hello']) } as any, {+ maxSize: 4+ })+ ).rejects.toEqual(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))++ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesLockManager.create).not.toHaveBeenCalled()+ })++ it('should stop an upload without content-length when streamed bytes exceed the remaining quota', async () => {+ const space = makeSpace({ storageQuota: 10, storageUsage: 9 })+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)+ vi.mocked(filesUtils.writeUploadFromStream).mockImplementationOnce(async (_path, _stream, options) => {+ options.limiter.consume(2)+ })++ await expect(service.saveStream(user, space, { method: 'PUT', headers: {}, raw: Readable.from(['xx']) } as any)).rejects.toEqual(+ new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)+ )++ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')+ })++ it('should short-circuit a known body above the remaining quota', async () => {+ const space = makeSpace({ storageQuota: 10, storageUsage: 9 })+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)++ await expect(+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-length': '2' }, raw: Readable.from(['xx']) } as any)+ ).rejects.toEqual(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))++ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesLockManager.create).not.toHaveBeenCalled()+ })+it('should use DAV conflict checks and checksum mode when requested', async () => {const space = makeSpace()setPathExists({ [space.realPath]: true, [path.dirname(space.realPath)]: true }, true)@@ -262,7 +307,70 @@ describe(FilesManager.name, () => {expect(checksum).toBe('sha256-abc')expect(filesLockManager.checkConflicts).toHaveBeenCalledWith(space.dbFile, DEPTH.RESOURCE, { userId: 7, lockTokens: ['token'] })expect(filesLockManager.create).not.toHaveBeenCalled()- expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStreamAndChecksum).toHaveBeenCalledWith(+ space.realPath,+ expect.anything(),+ 'sha256',+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 100, consume: expect.any(Function) }) })+ )+ })++ it('should accept a content range starting at zero for a new file', async () => {+ const space = makeSpace()+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)++ const result = await service.saveStream(+ user,+ space,+ { method: 'PUT', headers: { 'content-range': 'bytes 0-4/5' }, raw: Readable.from(['hello']) } as any,+ { dav: { depth: DEPTH.RESOURCE, lockTokens: [] } }+ )++ expect(result).toBe(false)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ space.realPath,+ expect.anything(),+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })+ )+ })++ it('should interpret the expected upload size as the final size of a resumed upload', async () => {+ const space = makeSpace({ storageQuota: 10, storageUsage: 8 })+ const tmpPath = '/data/users/john/tmp/sync-in-file.txt'+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true, [tmpPath]: true }, false)+ vi.mocked(filesUtils.fileSize).mockResolvedValueOnce(8)++ await expect(+ service.saveStream(+ user,+ space,+ {+ method: 'PUT',+ headers: { 'content-range': 'bytes 8-9/10', 'content-length': '2' },+ raw: Readable.from(['xx'])+ } as any,+ { tmpPath, expectedUploadSize: 10 }+ )+ ).resolves.toBe(false)++ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ tmpPath,+ expect.anything(),+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 8, consume: expect.any(Function) }) })+ )+ })++ it('should reject a non-zero content range when the file does not exist', async () => {+ const space = makeSpace()+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true }, false)++ await expect(+ service.saveStream(user, space, { method: 'PUT', headers: { 'content-range': 'bytes 5-9/10' }, raw: Readable.from(['hello']) } as any, {+ dav: { depth: DEPTH.RESOURCE, lockTokens: [] }+ })+ ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size'))++ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()})it('should validate tmp stream before moving it to the destination', async () => {@@ -282,7 +390,12 @@ describe(FilesManager.name, () => {).rejects.toEqual(validationError)expect(validateTmpFile).toHaveBeenCalledWith({ tmpPath, realPath: space.realPath, checksum: 'sha256-abc' })- expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalledWith(tmpPath, expect.anything(), 0, 'sha256')+ expect(filesUtils.writeUploadFromStreamAndChecksum).toHaveBeenCalledWith(+ tmpPath,+ expect.anything(),+ 'sha256',+ expect.objectContaining({ limiter: expect.objectContaining({ initialFileSize: 0, consume: expect.any(Function) }) })+ )expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(tmpPath)expect(emitSpy).not.toHaveBeenCalled()@@ -359,11 +472,15 @@ describe(FilesManager.name, () => {await service.saveMultipart(user, space, req as any)- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as stringexpect(filesLockManager.createOrRefresh).toHaveBeenCalled()expect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)- expect(tmpWritePath.endsWith('-report.txt')).toBe(true)- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, expect.anything())+ expect(path.basename(tmpWritePath)).toMatch(/^report\.txt-upload-/)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ tmpWritePath,+ expect.anything(),+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, '/data/users/john/files/report.txt', true)expect(emitSpy).toHaveBeenCalledWith('event',@@ -390,7 +507,7 @@ describe(FilesManager.name, () => {await expect(service.saveMultipart(user, space, req as any)).rejects.toEqual(new FileError(HttpStatus.NOT_FOUND, 'Location not found'))- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesUtils.removeFiles).not.toHaveBeenCalled()expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()@@ -413,9 +530,13 @@ describe(FilesManager.name, () => {await service.saveMultipart(user, space, req as any)- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as stringexpect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ tmpWritePath,+ file,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, space.realPath, true)expect(filesUtils.removeFiles).not.toHaveBeenCalled()expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.UPDATE, rPath: space.realPath }))@@ -451,11 +572,42 @@ describe(FilesManager.name, () => {await service.saveMultipart(user, space, req as any)expect(filesUtils.makeDir).toHaveBeenCalledWith(dstDir, true)- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(dstFile, file)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ dstFile,+ file,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-created')expect(emitSpy).toHaveBeenCalledWith('event', expect.objectContaining({ action: ACTION.ADD, rPath: dstFile }))})+ it('should share the remaining quota across multipart files', async () => {+ const space = makeSpace({ storageQuota: 5, storageUsage: 0 })+ const parentPath = path.dirname(space.realPath)+ const firstPath = path.join(parentPath, 'one.bin')+ const secondPath = path.join(parentPath, 'two.bin')+ setPathExists({ [space.realPath]: false, [parentPath]: true, [firstPath]: false, [secondPath]: false }, false)+ vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p === parentPath)+ vi.mocked(filesUtils.writeUploadFromStream).mockImplementation(async (_path, _stream, options) => {+ options.limiter.consume(3)+ })++ const req = {+ method: 'POST',+ files: async function* () {+ yield { filename: 'one.bin', file: Readable.from(['one']) }+ yield { filename: 'two.bin', file: Readable.from(['two']) }+ }+ }++ await expect(service.saveMultipart(user, space, req as any)).rejects.toEqual(+ new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)+ )++ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledTimes(2)+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(secondPath)+ })+it('should reject POST when resolved multipart destination already exists', async () => {const space = makeSpace()const partFileName = 'folder/file.txt'@@ -484,7 +636,7 @@ describe(FilesManager.name, () => {new FileError(HttpStatus.METHOD_NOT_ALLOWED, 'Resource already exists'))- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesUtils.removeFiles).not.toHaveBeenCalled()expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()@@ -510,7 +662,7 @@ describe(FilesManager.name, () => {expect(filesUtils.isPathExists).not.toHaveBeenCalledWith(forbiddenFile)expect(filesUtils.isPathIsDir).not.toHaveBeenCalledWith(forbiddenFile)- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()expect(emitSpy).not.toHaveBeenCalled()@@ -535,9 +687,13 @@ describe(FilesManager.name, () => {new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as stringexpect(tmpWritePath.startsWith(`${user.tmpPath}${path.sep}`)).toBe(true)- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ tmpWritePath,+ file,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpWritePath)expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(space.realPath)expect(filesUtils.moveFiles).not.toHaveBeenCalled()@@ -565,7 +721,11 @@ describe(FilesManager.name, () => {)expect(req.files).toHaveBeenCalled()- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(dstFile, file)+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ dstFile,+ file,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(filesUtils.removeFiles).toHaveBeenCalledWith(dstFile)expect(emitSpy).not.toHaveBeenCalled()})@@ -588,7 +748,7 @@ describe(FilesManager.name, () => {new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()})it('should not map non-file-size multipart 413 errors to file size limit', async () => {@@ -607,7 +767,7 @@ describe(FilesManager.name, () => {await expect(service.saveMultipart(user, space, req as any)).rejects.toBe(error)- expect(filesUtils.writeFromStream).not.toHaveBeenCalled()+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()})it.each([@@ -649,13 +809,17 @@ describe(FilesManager.name, () => {await expect(service.saveMultipart(user, space, req as any)).rejects.toBe(error)- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(tmpWritePath, file)+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as string+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(+ tmpWritePath,+ file,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(deleteSpy).toHaveBeenCalledTimes(1)expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, expect.stringContaining(path.basename(partFileName)), true)expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpWritePath)expect(emitSpy).not.toHaveBeenCalled()- expect(vi.mocked(filesUtils.writeFromStream).mock.invocationCallOrder[0]).toBeLessThan(deleteSpy.mock.invocationCallOrder[0])+ expect(vi.mocked(filesUtils.writeUploadFromStream).mock.invocationCallOrder[0]).toBeLessThan(deleteSpy.mock.invocationCallOrder[0])expect(deleteSpy.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.moveFiles).mock.invocationCallOrder[0])})@@ -685,7 +849,7 @@ describe(FilesManager.name, () => {await service.saveMultipart(user, space, req as any)- const tmpWritePath = vi.mocked(filesUtils.writeFromStream).mock.calls[0][0] as string+ const tmpWritePath = vi.mocked(filesUtils.writeUploadFromStream).mock.calls[0][0] as stringexpect(deleteSpy).toHaveBeenCalledTimes(1)expect(filesUtils.makeDir).toHaveBeenCalledWith(dstDir, true)expect(filesUtils.moveFiles).toHaveBeenCalledWith(tmpWritePath, dstFile, true)@@ -1070,13 +1234,10 @@ describe(FilesManager.name, () => {expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/download.txt')expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()- expect(filesUtils.writeFromStream).toHaveBeenCalledWith(+ expect(filesUtils.writeUploadFromStream).toHaveBeenCalledWith(taskPath('task-1', 'download.txt'),expect.anything(),- 0,- 55,- undefined,- expect.any(Function)+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }), onProgress: expect.any(Function) }))expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-1', 'download.txt'), '/tmp/download.txt')@@ -1090,12 +1251,32 @@ describe(FilesManager.name, () => {expect(fileEmitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })})+ it('should reject a remote download exceeding maxUploadSize before GET', async () => {+ const space = makeSpace()+ const tmpPath = '/data/users/john/tmp/download.txt-download-uuid'+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/download.txt')+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce(tmpPath)+ http.axiosRef.mockResolvedValueOnce({+ headers: { 'content-length': `${configuration.applications.files.maxUploadSize + 1}` },+ request: { socket: { remoteAddress: '8.8.8.8' } }+ })++ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toEqual(+ new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)+ )++ expect(http.axiosRef).toHaveBeenCalledTimes(1)+ expect(filesUtils.writeUploadFromStream).not.toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith(tmpPath)+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')+ })+it('should cleanup partial file and skip ADD event when download write fails', async () => {const error = new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED)const space = makeSpace()vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/download.txt')vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')- vi.mocked(filesUtils.writeFromStream).mockRejectedValueOnce(error)+ vi.mocked(filesUtils.writeUploadFromStream).mockRejectedValueOnce(error)http.axiosRef.mockResolvedValueOnce({headers: { 'content-length': '55' },diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex c9dc8fc0..38c98e38 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -5,6 +5,7 @@ import { Readable } from 'node:stream'import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'import { generateThumbnail } from '../../../common/image'import { SERVER_NAME } from '../../../common/shared'+import { configuration } from '../../../configuration/config.environment'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'import { HTTP_METHOD } from '../../applications.constants'import { NOTIFICATION_APP, NOTIFICATION_APP_EVENT } from '../../notifications/constants/notifications'@@ -47,11 +48,12 @@ import {moveFiles,removeFiles,tempFilePath,+ temporaryPathPrefix,touchFile,uniqueDatedFilePath,uniqueFilePathFromDir,- writeFromStream,- writeFromStreamAndChecksum+ writeUploadFromStream,+ writeUploadFromStreamAndChecksum} from '../utils/files'import { SendFile } from '../utils/send-file'import { extractZip } from '../utils/unzip-file'@@ -61,7 +63,13 @@ import { FilesLockManager } from './files-lock-manager.service'import { FilesQueries } from './files-queries.service'import { FileEvent, FileTaskEvent } from '../events/file-events'import { ACTION } from '../../../common/constants'-import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'+import {+ createUploadStreamLimiter,+ isMultipartFileTooLargeError,+ parseContentLength,+ parseContentRange,+ uploadTmpFilePath+} from '../utils/upload-file'import { maxFileSizeExceededError } from '../utils/errors'import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'@@ -71,7 +79,7 @@ import { FILE_ERROR } from '../constants/errors'@Injectable()export class FilesManager {- /* Spaces permissions are checked in the space guard, except for the copy/move destination */+ // Spaces permissions are checked in the space guard, except for the copy/move destinationprivate logger = new Logger(FilesManager.name)constructor(@@ -107,13 +115,41 @@ export class FilesManager {if (fExists && (await isPathIsDir(space.realPath))) {throw new FileError(HttpStatus.METHOD_NOT_ALLOWED, 'The location is a directory')}++ // Resolve the final file-size boundary and reject a known oversized body+ // before creating directories or acquiring a lock.+ let startRange = 0+ const contentRange = req.headers['content-range']+ if (contentRange !== undefined) {+ // With PUT method, some webdav clients use the `content-range` header,+ // which is normally reserved for a response to a request containing the `range` header.+ // However, for more compatibility let's accept it.+ startRange = parseContentRange(contentRange).start+ // Ranged PUTs are append-only resumes. A missing file has size zero, so it can only+ // be created by a first range starting at offset zero.+ const currentSize = fExists || fTmpExists ? await fileSize(options?.tmpPath || space.realPath) : 0+ if (startRange !== currentSize) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size')+ }+ }+ const maxUploadSize = configuration.applications.files.maxUploadSize+ const maxSize = Math.min(maxUploadSize, options?.maxSize ?? maxUploadSize)+ const fileLimiter = createUploadStreamLimiter(space, maxSize).createFileLimiter(startRange)+ const contentLength = parseContentLength(req.headers['content-length'])+ if (contentLength !== undefined) {+ fileLimiter.assertKnownSize(contentLength)+ }+ if (options?.expectedUploadSize !== undefined) {+ fileLimiter.assertFinalSize(options.expectedUploadSize)+ }+if (options?.tmpPath) {// Ensure tmpPath parent dir existsawait makeDir(dirName(options.tmpPath), true)} else if (!(await isPathExists(dirName(space.realPath)))) {throw new FileError(HttpStatus.CONFLICT, 'Parent must exists')}- /* File Lock */+ // File Locklet fileLock: FileLock | undefinedif (options?.dav) {// Check locks@@ -132,29 +168,17 @@ export class FilesManager {const fileEventAction = fExists ? ACTION.UPDATE : ACTION.ADDlet fileWritten = falsetry {- // Check range- let startRange = 0- if ((fExists || fTmpExists) && req.headers['content-range']) {- // With PUT method, some webdav clients use the `content-range` header,- // which is normally reserved for a response to a request containing the `range` header.- // However, for more compatibility let's accept it.- const match = /\d+/.exec(req.headers['content-range'])- if (!match.length) {- throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')- }- startRange = parseInt(match[0], 10)- const size = await fileSize(options?.tmpPath || space.realPath)- if (startRange !== size) {- throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : start offset does not match the current file size')- }- }// todo: check file in db to update// todo : versioning herelet checksum: stringif (options?.checksumAlg) {- checksum = await writeFromStreamAndChecksum(options?.tmpPath || space.realPath, req.raw, startRange, options.checksumAlg)+ checksum = await writeUploadFromStreamAndChecksum(options?.tmpPath || space.realPath, req.raw, options.checksumAlg, {+ limiter: fileLimiter+ })} else {- await writeFromStream(options?.tmpPath || space.realPath, req.raw, startRange)+ await writeUploadFromStream(options?.tmpPath || space.realPath, req.raw, {+ limiter: fileLimiter+ })}if (options?.tmpPath) {await options.validateTmpFile?.({ tmpPath: options.tmpPath, realPath: space.realPath, checksum })@@ -205,6 +229,7 @@ export class FilesManager {const patchMethod = req.method === HTTP_METHOD.PATCHconst postMethod = req.method === HTTP_METHOD.POSTconst realParentPath = dirName(space.realPath)+ const uploadLimiter = createUploadStreamLimiter(space, configuration.applications.files.maxUploadSize)// For POST, space.realPath can be either the final file path or the root directory for a folder upload.if (postMethod && (await isPathExists(space.realPath))) {@@ -272,7 +297,8 @@ export class FilesManager {let fileWritten = false// Dotry {- await writeFromStream(writePath, part.file)+ const fileLimiter = uploadLimiter.createFileLimiter()+ await writeUploadFromStream(writePath, part.file, { limiter: fileLimiter })// With throwFileSizeLimit disabled, multipart marks the file stream as truncated instead of rejecting.if (part.file.truncated) {throw maxFileSizeExceededError()@@ -456,7 +482,7 @@ export class FilesManager {const isDir = await isPathIsDir(srcSpace.realPath)if (dstSpace.storageQuota) {- /* Skip validation when moving to the same space; for copy operations, run all checks. */+ // Skip validation when moving to the same space; for copy operations, run all checks.if (!isMove || (isMove && srcSpace.id !== dstSpace.id)) {const size = isDir ? (await dirSize(srcSpace.realPath))[0] : await fileSize(srcSpace.realPath)if (dstSpace.willExceedQuota(size)) {@@ -602,7 +628,7 @@ export class FilesManager {const dstPath = await uniqueFilePathFromDir(space.realPath)const tmpPath = isTaskContext? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)- : tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)+ : tempFilePath(user.tmpPath, temporaryPathPrefix(dstPath, 'download'))const dbFile = space.dbFiledbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))@@ -616,6 +642,7 @@ export class FilesManager {await new DownloadFile(this.http).download(downloadDto, tmpPath, {space,publishedPath: dstPath,+ maxSize: configuration.applications.files.maxUploadSize,signal,onProgress: isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined})@@ -646,7 +673,7 @@ export class FilesManager {const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))const tmpPath = isTaskContext? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)- : tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)+ : tempFilePath(user.tmpPath, temporaryPathPrefix(dstPath, 'compress'))// create locklet fileLock: FileLock | undefinedif (dto.compressInDirectory) {@@ -701,7 +728,7 @@ export class FilesManager {const dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))const tmpPath = isTaskContext? await createTaskTemporaryDir(user.tasksPath, space.task!.cacheKey, dstPath)- : await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)+ : await makeTempDir(user.tmpPath, temporaryPathPrefix(dstPath, 'extract'))let fileLock: FileLock | undefinedtry {// create lockdiff --git a/backend/src/applications/files/utils/download-file.spec.ts b/backend/src/applications/files/utils/download-file.spec.tsindex 566a0150..08b3b6ad 100644--- a/backend/src/applications/files/utils/download-file.spec.ts+++ b/backend/src/applications/files/utils/download-file.spec.ts@@ -1,16 +1,17 @@import { lookup } from 'node:dns/promises'import { HttpService } from '@nestjs/axios'-import { HttpStatus } from '@nestjs/common'+import { HttpStatus, Logger } from '@nestjs/common'import { Readable } from 'node:stream'import { HTTP_METHOD } from '../../applications.constants'import { FileError } from '../models/file-error'-import { writeFromStream } from './files'+import { writeUploadFromStream } from './files'import { DownloadFile } from './download-file'import type { Mock } from 'vitest'import { FILE_ERROR } from '../constants/errors'vi.mock('./files', () => ({- writeFromStream: vi.fn()+ writeUploadFromStream: vi.fn(),+ temporaryPathPrefix: vi.fn()}))vi.mock('node:dns/promises', () => ({lookup: vi.fn()@@ -44,13 +45,15 @@ describe(DownloadFile.name, () => {})beforeEach(() => {+ vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined)http = { axiosRef: vi.fn() }lookupMock.mockResolvedValue([{ address: '8.8.8.8', family: 4 }])- vi.mocked(writeFromStream).mockResolvedValue(undefined)+ vi.mocked(writeUploadFromStream).mockResolvedValue(undefined)})afterEach(() => {vi.clearAllMocks()+ vi.restoreAllMocks()})it.each(blockedRemoteAddresses)('rejects blocked remote address "%s" on HEAD by default', async (remoteAddress) => {@@ -64,7 +67,7 @@ describe(DownloadFile.name, () => {expect(http.axiosRef).toHaveBeenCalledWith(expect.objectContaining({ method: HTTP_METHOD.HEAD, url: 'https://example.test/file.txt', maxRedirects: 0 }))- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('rejects blocked DNS resolutions before calling HEAD', async () => {@@ -76,7 +79,7 @@ describe(DownloadFile.name, () => {expect(lookupMock).toHaveBeenCalledWith('example.test', { all: true, order: 'verbatim' })expect(http.axiosRef).not.toHaveBeenCalled()- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('rejects mixed DNS resolutions when one address is blocked', async () => {@@ -90,7 +93,7 @@ describe(DownloadFile.name, () => {).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))expect(http.axiosRef).not.toHaveBeenCalled()- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('follows a safe redirect manually', async () => {@@ -115,7 +118,7 @@ describe(DownloadFile.name, () => {2,expect.objectContaining({ method: HTTP_METHOD.HEAD, url: 'https://cdn.example.test/file.txt', maxRedirects: 0 }))- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('rejects a redirect to a blocked DNS resolution before following it', async () => {@@ -127,7 +130,7 @@ describe(DownloadFile.name, () => {).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))expect(http.axiosRef).toHaveBeenCalledTimes(1)- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('rejects a missing remote address on HEAD by default', async () => {@@ -138,7 +141,7 @@ describe(DownloadFile.name, () => {).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))expect(http.axiosRef).toHaveBeenCalledTimes(1)- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it.each(publicRemoteAddresses)('allows public remote address "%s" on HEAD by default', async (remoteAddress) => {@@ -161,7 +164,7 @@ describe(DownloadFile.name, () => {httpsAgent: expect.any(Object)}))- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('uses a guarded agent lookup for connection-time resolutions', async () => {@@ -187,7 +190,7 @@ describe(DownloadFile.name, () => {new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', { getContentInfo: true })).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('allows private IPs on HEAD when allowPrivateIP is enabled for content info', async () => {@@ -210,7 +213,7 @@ describe(DownloadFile.name, () => {lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT'})expect(http.axiosRef).toHaveBeenCalledTimes(1)- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('rejects private IPs on GET by default and closes the stream', async () => {@@ -225,7 +228,7 @@ describe(DownloadFile.name, () => {).rejects.toEqual(new FileError(HttpStatus.FORBIDDEN, FILE_ERROR.DOWNLOAD_PRIVATE_IP))expect(destroySpy).toHaveBeenCalled()- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it.each(['', '-1', '1.5', 'Infinity', '9007199254740992', 'abc'])('rejects invalid content-length "%s"', async (contentLength) => {@@ -236,7 +239,18 @@ describe(DownloadFile.name, () => {).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))expect(http.axiosRef).toHaveBeenCalledTimes(1)- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()+ })++ it('rejects a missing content-length when no explicit maximum is available', async () => {+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8'))++ await expect(+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')+ ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))++ expect(http.axiosRef).toHaveBeenCalledTimes(1)+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('allows missing content-length when maxSize is provided', async () => {@@ -247,27 +261,30 @@ describe(DownloadFile.name, () => {maxSize: 1024})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )})- it('rejects missing content-length for space downloads even when maxSize is provided', async () => {- const space = { willExceedQuota: vi.fn() }- http.axiosRef.mockResolvedValueOnce(response('8.8.8.8'))+ it('allows missing content-length for space downloads when maxSize is provided', async () => {+ const stream = Readable.from(['abc'])+ const space = { storageQuota: 10, storageUsage: 7 }+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8')).mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })- await expect(- new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {- space: space as any,- maxSize: 1024- })- ).rejects.toEqual(new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH))+ await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {+ space: space as any,+ maxSize: 1024+ })- expect(space.willExceedQuota).not.toHaveBeenCalled()- expect(writeFromStream).not.toHaveBeenCalled()+ const options = vi.mocked(writeUploadFromStream).mock.calls[0][2]+ expect(() => options.limiter.consume(4)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))})- it('keeps content-length as the stream guard for space downloads even when maxSize is provided', async () => {+ it('accepts a known content-length within the remaining quota', async () => {const stream = Readable.from(['abc'])- const space = { willExceedQuota: vi.fn().mockReturnValue(false) }+ const space = { storageQuota: 100, storageUsage: 88 }http.axiosRef.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' })).mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })@@ -277,8 +294,41 @@ describe(DownloadFile.name, () => {maxSize: 1024})- expect(space.willExceedQuota).toHaveBeenCalledWith(12)- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )+ })++ it('rejects a known content-length above the remaining quota before GET', async () => {+ const space = { storageQuota: 100, storageUsage: 89 }+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' }))++ await expect(+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {+ space: space as any,+ maxSize: 1024+ })+ ).rejects.toEqual(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))++ expect(http.axiosRef).toHaveBeenCalledTimes(1)+ expect(writeUploadFromStream).not.toHaveBeenCalled()+ })++ it('rejects a space download exceeding maxSize before starting the GET request', async () => {+ const space = { storageQuota: null, storageUsage: 0 }+ http.axiosRef.mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '1025' }))++ await expect(+ new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {+ space: space as any,+ maxSize: 1024+ })+ ).rejects.toEqual(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))++ expect(http.axiosRef).toHaveBeenCalledTimes(1)+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('forwards the progress callback to the stream writer', async () => {@@ -292,7 +342,11 @@ describe(DownloadFile.name, () => {onProgress})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, onProgress)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }), onProgress })+ )})it('allows zero content-length and guards the written stream at zero bytes', async () => {@@ -303,7 +357,11 @@ describe(DownloadFile.name, () => {await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0, undefined, undefined)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )})it('rejects redirects on GET after the HEAD URL has been resolved', async () => {@@ -319,7 +377,7 @@ describe(DownloadFile.name, () => {expect(http.axiosRef).toHaveBeenCalledTimes(2)expect(destroySpy).toHaveBeenCalled()- expect(writeFromStream).not.toHaveBeenCalled()+ expect(writeUploadFromStream).not.toHaveBeenCalled()})it('allows private IPs on GET when allowPrivateIP is enabled', async () => {@@ -332,7 +390,11 @@ describe(DownloadFile.name, () => {allowPrivateIP: true})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )expect(http.axiosRef).toHaveBeenNthCalledWith(2,expect.objectContaining({@@ -357,6 +419,10 @@ describe(DownloadFile.name, () => {maxSize: 1024})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)+ expect(writeUploadFromStream).toHaveBeenCalledWith(+ '/tmp/file.txt',+ stream,+ expect.objectContaining({ limiter: expect.objectContaining({ consume: expect.any(Function) }) })+ )})})diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.tsindex 5965c397..105ff4ec 100644--- a/backend/src/applications/files/utils/download-file.ts+++ b/backend/src/applications/files/utils/download-file.ts@@ -4,7 +4,7 @@ import { Agent as HttpsAgent } from 'node:https'import type { LookupAddress } from 'node:dns'import type { LookupFunction } from 'node:net'import type { HttpService } from '@nestjs/axios'-import { HttpStatus } from '@nestjs/common'+import { HttpStatus, Logger } from '@nestjs/common'import { AxiosHeaders, type AxiosRequestConfig, type AxiosResponse } from 'axios'import ipaddr from 'ipaddr.js'import { HTTP_METHOD } from '../../applications.constants'@@ -13,15 +13,19 @@ import type { DownloadFileDto } from '../dto/file-operations.dto'import { FileTaskEvent } from '../events/file-events'import type { DownloadFileContentInfo, DownloadFileOptions } from '../interfaces/download-file.interface'import { FileError } from '../models/file-error'-import { writeFromStream } from './files'+import { writeUploadFromStream } from './files'import { FILE_ERROR } from '../constants/errors'+import { createUploadStreamLimiter } from './upload-file'interface DownloadFileRequestOptions {allowPrivateIP?: booleanmaxRedirects?: number}+type DownloadStage = 'head' | 'validation' | 'get' | 'stream'+export class DownloadFile {+ private readonly logger = new Logger(DownloadFile.name)private static readonly dnsOptions = { all: true, order: 'verbatim' } as constprivate static readonly maxRedirects = 1private static readonly redirectStatus = new Set([301, 302, 303, 307, 308])@@ -41,36 +45,76 @@ export class DownloadFile {async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions & { getContentInfo?: false | undefined }): Promise<void>async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions): Promise<void | DownloadFileContentInfo> {const identityEncodingConfig = { decompress: false, headers: { 'Accept-Encoding': 'identity' } }+ let stage: DownloadStage = 'head'+ let currentUrl = downloadDto.url+ let contentLength: number | null | undefined+ let maxSize: number | undefined++ try {+ const { response: headRes, url } = await this.request(+ currentUrl,+ { method: HTTP_METHOD.HEAD, signal: options?.signal, ...identityEncodingConfig },+ { allowPrivateIP: options?.allowPrivateIP }+ )+ currentUrl = url++ stage = 'validation'+ const headers = AxiosHeaders.from(headRes.headers)+ contentLength = this.contentLength(headers)+ if (options?.getContentInfo) {+ return {+ contentLength,+ contentType: `${headers.getContentType()}`,+ lastModified: headers.get('last-modified') as string | undefined+ } satisfies DownloadFileContentInfo+ }- const { response: headRes, url } = await this.request(- downloadDto.url,- { method: HTTP_METHOD.HEAD, signal: options?.signal, ...identityEncodingConfig },- { allowPrivateIP: options?.allowPrivateIP }- )-- const headers = AxiosHeaders.from(headRes.headers)- const contentLength = this.contentLength(headers)- if (options?.getContentInfo) {- return {- contentLength,- contentType: `${headers.getContentType()}`,- lastModified: headers.get('last-modified') as string | undefined- } satisfies DownloadFileContentInfo+ // An explicit maximum allows chunked downloads; otherwise HEAD content-length is the stream boundary.+ const streamMaxSize = options?.maxSize ?? contentLength+ if (streamMaxSize === null || streamMaxSize === undefined) {+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)+ }+ maxSize = streamMaxSize+ const fileLimiter = createUploadStreamLimiter(options?.space, streamMaxSize).createFileLimiter()+ if (contentLength !== null) {+ fileLimiter.assertKnownSize(contentLength)+ }+ this.prepareSpaceTask(options?.space, contentLength, options?.publishedPath || dstPath)++ // The HEAD request resolved redirects; the GET must target that final URL directly.+ stage = 'get'+ const { response: getRes } = await this.request(+ currentUrl,+ { method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },+ { allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 }+ )+ stage = 'stream'+ try {+ const getContentLength = this.contentLength(AxiosHeaders.from(getRes.headers))+ if (getContentLength !== null) {+ fileLimiter.assertKnownSize(getContentLength)+ }+ } catch (e) {+ getRes.data?.destroy?.()+ throw e+ }+ await writeUploadFromStream(dstPath, getRes.data, {+ limiter: fileLimiter,+ signal: options?.signal,+ onProgress: options?.onProgress+ })+ } catch (e) {+ this.logFailure(stage, currentUrl, e, options?.signal, contentLength, maxSize ?? options?.maxSize)+ throw e}+ }- const maxSize = options?.space ? contentLength : (options?.maxSize ?? contentLength)- if (maxSize === null || (contentLength === null && options?.space)) {- throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)- }- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath)-- // The HEAD request resolved redirects; the GET must target that final URL directly.- const { response: getRes } = await this.request(- url,- { method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },- { allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 }- )- await writeFromStream(dstPath, getRes.data, 0, maxSize, options?.signal, options?.onProgress)+ private logFailure(stage: DownloadStage, url: string, error: unknown, signal?: AbortSignal, contentLength?: number | null, maxSize?: number): void {+ if (signal?.aborted) return+ const host = URL.canParse(url) ? new URL(url).hostname || 'unknown' : 'invalid'+ const sizeContext =+ stage === 'validation' || stage === 'stream' ? ` (contentLength=${contentLength ?? 'missing'}, maxSize=${maxSize ?? 'unset'})` : ''+ this.logger.warn({ tag: this.download.name, msg: `${stage} failed for host *${host}*${sizeContext}: ${error}` })}private safeLookup(hostname: string, options: Parameters<LookupFunction>[1], cb: Parameters<LookupFunction>[2]): void {@@ -163,18 +207,29 @@ export class DownloadFile {}private contentLength(headers: AxiosHeaders): number | null {- const value = headers.getContentLength(/^\d+$/)?.[0]- const contentLength = Number(value)- return Number.isSafeInteger(contentLength) ? contentLength : null+ const value = headers.get('content-length')+ if (value === undefined || value === null) return null++ const normalized = typeof value === 'number' ? String(value) : typeof value === 'string' ? value.trim() : ''+ if (!/^\d+$/.test(normalized)) {+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)+ }+ const contentLength = Number(normalized)+ if (!Number.isSafeInteger(contentLength)) {+ throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR.DOWNLOAD_INVALID_CONTENT_LENGTH)+ }+ return contentLength}- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string): void {+ private prepareSpaceTask(space: SpaceEnv | undefined, contentLength: number | null, publishedPath: string): void {if (!space) return- if (space.willExceedQuota(contentLength)) {- throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)- }if (space.task?.cacheKey) {- space.task.props = { ...space.task.props, progress: 1, size: 0, totalSize: contentLength }+ space.task.props = {+ ...space.task.props,+ progress: 1,+ size: 0,+ totalSize: contentLength ?? undefined+ }FileTaskEvent.emit('startWatch', space, publishedPath)}}diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.tsindex 865de371..1ad5a33f 100644--- a/backend/src/applications/files/utils/files.spec.ts+++ b/backend/src/applications/files/utils/files.spec.ts@@ -7,8 +7,21 @@ import fse from 'fs-extra'import type { MockInstance } from 'vitest'import { FileError } from '../models/file-error'import { storageQuotaExceededError } from './errors'-import { createSizeLimiter, isCrossDevice, isPathInside, makeTempDir, sanitizeName, tempFilePath, writeFromStream } from './files'+import {+ createSizeLimiter,+ isCrossDevice,+ isPathInside,+ makeTempDir,+ sanitizeName,+ tempFilePath,+ temporaryPathPrefix,+ writeFromStream,+ writeFromStreamAndChecksum,+ writeUploadFromStream,+ writeUploadFromStreamAndChecksum+} from './files'import { FILE_ERROR } from '../constants/errors'+import { UploadStreamLimiter } from './upload-file'describe(createSizeLimiter.name, () => {it('rejects the call that makes the cumulative size exceed the limit', () => {@@ -81,35 +94,118 @@ describe(writeFromStream.name, () => {await rm(tmpDir, { recursive: true, force: true })})- it('writes from an offset up to the max size and reports progress', async () => {+ it('writes from an offset and accounts each chunk', async () => {+ const filePath = path.join(tmpDir, 'file.txt')+ const accountBytes = vi.fn()+ await writeFile(filePath, 'abc')++ await writeFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), { start: 3, accountBytes })++ await expect(readFile(filePath, 'utf8')).resolves.toBe('abcdef')+ expect(accountBytes).toHaveBeenNthCalledWith(1, 2)+ expect(accountBytes).toHaveBeenNthCalledWith(2, 1)+ })++ it('propagates a byte accounting error', async () => {+ const filePath = path.join(tmpDir, 'file.txt')++ await expect(+ writeFromStream(filePath, Readable.from([Buffer.from('abcd')]), {+ accountBytes: () => {+ throw storageQuotaExceededError()+ }+ })+ ).rejects.toMatchObject({+ httpCode: HttpStatus.INSUFFICIENT_STORAGE,+ message: FILE_ERROR.STORAGE_QUOTA_EXCEEDED,+ name: FileError.name+ })+ })++ it('aborts a stream when its signal is already aborted', async () => {+ const filePath = path.join(tmpDir, 'file.txt')+ const controller = new AbortController()+ controller.abort()++ await expect(writeFromStream(filePath, Readable.from([Buffer.from('abc')]), { signal: controller.signal })).rejects.toMatchObject({+ name: 'AbortError'+ })+ })+})++describe(writeFromStreamAndChecksum.name, () => {+ let tmpDir: string++ beforeEach(async () => {+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'write-from-stream-checksum-'))+ })++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true })+ })++ it('propagates a byte accounting error in checksum mode', async () => {+ const filePath = path.join(tmpDir, 'file.txt')++ await expect(+ writeFromStreamAndChecksum(filePath, Readable.from([Buffer.from('abcd')]), 'sha256', {+ accountBytes: () => {+ throw storageQuotaExceededError()+ }+ })+ ).rejects.toMatchObject({+ httpCode: HttpStatus.INSUFFICIENT_STORAGE,+ message: FILE_ERROR.STORAGE_QUOTA_EXCEEDED,+ name: FileError.name+ })+ })+})++describe('upload stream writers', () => {+ let tmpDir: string++ beforeEach(async () => {+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'write-upload-from-stream-'))+ })++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true })+ })++ it('uses the limiter offset and reports only accepted chunks', async () => {const filePath = path.join(tmpDir, 'file.txt')const onProgress = vi.fn()+ const limiter = new UploadStreamLimiter(6).createFileLimiter(3)await writeFile(filePath, 'abc')- await writeFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), 3, 6, undefined, onProgress)+ await writeUploadFromStream(filePath, Readable.from([Buffer.from('de'), Buffer.from('f')]), { limiter, onProgress })await expect(readFile(filePath, 'utf8')).resolves.toBe('abcdef')expect(onProgress).toHaveBeenNthCalledWith(1, 2)expect(onProgress).toHaveBeenNthCalledWith(2, 1)})- it('rejects a stream exceeding the max size', async () => {+ it('does not report a chunk rejected by the limiter', async () => {const filePath = path.join(tmpDir, 'file.txt')+ const onProgress = vi.fn()+ const limiter = new UploadStreamLimiter(3).createFileLimiter()- await expect(writeFromStream(filePath, Readable.from([Buffer.from('abcd')]), 0, 3)).rejects.toMatchObject({+ await expect(writeUploadFromStream(filePath, Readable.from([Buffer.from('abcd')]), { limiter, onProgress })).rejects.toMatchObject({httpCode: HttpStatus.PAYLOAD_TOO_LARGE,message: FILE_ERROR.MAX_FILE_SIZE_EXCEEDED,name: FileError.name})+ expect(onProgress).not.toHaveBeenCalled()})- it('aborts a stream when its signal is already aborted', async () => {+ it('enforces the limiter in checksum mode', async () => {const filePath = path.join(tmpDir, 'file.txt')- const controller = new AbortController()- controller.abort()+ const limiter = new UploadStreamLimiter(3).createFileLimiter()- await expect(writeFromStream(filePath, Readable.from([Buffer.from('abc')]), 0, undefined, controller.signal)).rejects.toMatchObject({- name: 'AbortError'+ await expect(writeUploadFromStreamAndChecksum(filePath, Readable.from([Buffer.from('abcd')]), 'sha256', { limiter })).rejects.toMatchObject({+ httpCode: HttpStatus.PAYLOAD_TOO_LARGE,+ message: FILE_ERROR.MAX_FILE_SIZE_EXCEEDED,+ name: FileError.name})})})@@ -148,3 +244,11 @@ describe(tempFilePath.name, () => {expect(path.dirname(tempFilePath(parentPath, path.join('..', 'archive-')))).toBe(parentPath)})})++describe(temporaryPathPrefix.name, () => {+ it('uses the destination name and operation', () => {+ expect(temporaryPathPrefix('/destination/report.pdf', 'upload')).toBe('report.pdf-upload-')+ expect(temporaryPathPrefix('/destination', 'extract')).toBe('destination-extract-')+ expect(temporaryPathPrefix('', 'download')).toBe('file-download-')+ })+})diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex 0c9714fc..f9b460f1 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -15,6 +15,7 @@ import type { FileDBProps } from '../interfaces/file-db-props.interface'import type { FileProps } from '../interfaces/file-props.interface'import { FileError } from '../models/file-error'import { maxFileSizeExceededError } from './errors'+import type { UploadFileStreamLimiter } from './upload-file'export function sanitizePath(fPath: string): string {return path.normalize(fPath).replace(regExpPreventPathTraversal, '')@@ -118,6 +119,10 @@ export function tempFilePath(parentPath: string, prefix: string): string {return path.join(parentPath, `${path.basename(prefix)}${crypto.randomUUID()}`)}+export function temporaryPathPrefix(targetPath: string, operation: 'upload' | 'download' | 'compress' | 'extract'): string {+ return `${fileName(targetPath) || 'file'}-${operation}-`+}+export function getMimeType(fPath: string, isDir: boolean): string {if (isDir) {return 'directory'@@ -242,54 +247,90 @@ export function createProgressTransform(})}-export function writeFromStream(- rPath: string,- stream: Readable,- start: number = 0,- maxSize?: number,- signal?: AbortSignal,- onProgress?: (bytes: number) => void-): Promise<void> {+export interface WriteFromStreamOptions {+ start?: number+ signal?: AbortSignal+ // Called before forwarding a chunk to the destination. Throwing aborts the pipeline.+ accountBytes?: (bytes: number) => void+}++export function writeFromStream(rPath: string, stream: Readable, options: WriteFromStreamOptions = {}): Promise<void> {+ const { start = 0, signal, accountBytes } = optionsconst dst: WriteStream = createWriteStream(rPath, { flags: start ? 'a' : 'w', start: start, highWaterMark: DEFAULT_HIGH_WATER_MARK })- if (maxSize === undefined && !onProgress) {+ if (!accountBytes) {return pipeline(stream, dst, { signal })}- let received = start- const progress = new Transform({+ const accounting = new Transform({transform(chunk, _encoding, callback) {- received += chunk.length- if (maxSize !== undefined && received > maxSize) {- callback(maxFileSizeExceededError())+ try {+ accountBytes(chunk.length)+ } catch (error) {+ callback(error as Error)return}- onProgress?.(chunk.length)callback(null, chunk)}})- return pipeline(stream, progress, dst, { signal })+ return pipeline(stream, accounting, dst, { signal })}-export async function writeFromStreamAndChecksum(rPath: string, stream: Readable, hasRange: number, alg: string): Promise<string> {+export async function writeFromStreamAndChecksum(+ rPath: string,+ stream: Readable,+ alg: string,+ options: WriteFromStreamOptions = {}+): Promise<string> {+ const { start = 0, signal, accountBytes } = optionsconst hash = crypto.createHash(alg)- if (hasRange) {+ if (start) {+ // Seed the hash with the existing prefix so the result covers the complete resumed file.const src = createReadStream(rPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- await pipeline(src, hash, { end: false })+ await pipeline(src, hash, { end: false, signal })}- const dst = createWriteStream(rPath, { flags: hasRange ? 'a' : 'w', highWaterMark: DEFAULT_HIGH_WATER_MARK })+ const dst = createWriteStream(rPath, { flags: start ? 'a' : 'w', highWaterMark: DEFAULT_HIGH_WATER_MARK })await pipeline(stream,async function* (source) {for await (const chunk of source) {+ accountBytes?.(chunk.length)hash.update(chunk)yield chunk}},- dst+ dst,+ { signal })hash.end()return hash.digest('hex')}+export interface WriteUploadStreamOptions {+ limiter: UploadFileStreamLimiter+ signal?: AbortSignal+ // Receives only bytes accepted by the limiter.+ onProgress?: (bytes: number) => void+}++function uploadWriteOptions({ limiter, signal, onProgress }: WriteUploadStreamOptions): WriteFromStreamOptions {+ return {+ start: limiter.initialFileSize,+ signal,+ accountBytes: (bytes) => {+ // Progress must only include chunks accepted by both the file-size and quota boundaries.+ limiter.consume(bytes)+ onProgress?.(bytes)+ }+ }+}++export function writeUploadFromStream(rPath: string, stream: Readable, options: WriteUploadStreamOptions): Promise<void> {+ return writeFromStream(rPath, stream, uploadWriteOptions(options))+}++export function writeUploadFromStreamAndChecksum(rPath: string, stream: Readable, alg: string, options: WriteUploadStreamOptions): Promise<string> {+ return writeFromStreamAndChecksum(rPath, stream, alg, uploadWriteOptions(options))+}+export function copyFileContent(srcPath: string, dstPath: string): Promise<void> {const srcStream = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })return writeFromStream(dstPath, srcStream)diff --git a/backend/src/applications/files/utils/upload-file.spec.ts b/backend/src/applications/files/utils/upload-file.spec.tsnew file mode 100644index 00000000..f8dee0d2--- /dev/null+++ b/backend/src/applications/files/utils/upload-file.spec.ts@@ -0,0 +1,79 @@+import { HttpStatus } from '@nestjs/common'+import { FILE_ERROR } from '../constants/errors'+import { FileError } from '../models/file-error'+import { parseContentLength, parseContentRange, UploadStreamLimiter } from './upload-file'++describe(UploadStreamLimiter.name, () => {+ it('short-circuits a known file size above maxUploadSize', () => {+ const fileLimiter = new UploadStreamLimiter(10).createFileLimiter()++ expect(() => fileLimiter.assertKnownSize(11)).toThrow(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))+ })++ it('short-circuits a known file size above the remaining quota', () => {+ const fileLimiter = new UploadStreamLimiter(100, 4).createFileLimiter()++ expect(() => fileLimiter.assertKnownSize(5)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))+ })++ it('checks each streamed chunk before accepting it', () => {+ const fileLimiter = new UploadStreamLimiter(5).createFileLimiter()++ fileLimiter.consume(3)+ expect(() => fileLimiter.consume(3)).toThrow(new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR.MAX_FILE_SIZE_EXCEEDED))+ })++ it('shares quota consumption across multipart files', () => {+ const limiter = new UploadStreamLimiter(100, 6)+ limiter.createFileLimiter().consume(4)++ const secondFile = limiter.createFileLimiter()+ secondFile.consume(2)+ expect(() => secondFile.consume(1)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))+ })++ it('includes a range offset in maxUploadSize but not in newly consumed quota', () => {+ const limiter = new UploadStreamLimiter(10, 5)+ const rangedFile = limiter.createFileLimiter(8)++ expect(rangedFile.initialFileSize).toBe(8)+ rangedFile.assertFinalSize(10)+ rangedFile.consume(2)+ const secondFile = limiter.createFileLimiter()+ expect(() => secondFile.consume(3)).not.toThrow()+ expect(() => secondFile.consume(1)).toThrow(new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED))+ })++ it('rejects a final size below the current range offset', () => {+ const rangedFile = new UploadStreamLimiter(10).createFileLimiter(8)++ expect(() => rangedFile.assertFinalSize(7)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size'))+ })+})++describe(parseContentLength.name, () => {+ it('returns undefined when the header is absent', () => {+ expect(parseContentLength(undefined)).toBeUndefined()+ })++ it('parses a valid byte count', () => {+ expect(parseContentLength(' 42 ')).toBe(42)+ })++ it.each(['-1', '1.5', 'abc', '9007199254740992'])('rejects invalid content-length "%s"', (value) => {+ expect(() => parseContentLength(value)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header'))+ })+})++describe(parseContentRange.name, () => {+ const malformedRanges = ['bytes 5-4/10', 'bytes 0-10/10', 'bytes nope', 'bytes 9007199254740992-9007199254740993/*']++ it('parses ranges with a known or unknown total size', () => {+ expect(parseContentRange('bytes 5-9/10')).toEqual({ start: 5, end: 9, total: 10 })+ expect(parseContentRange('bytes 5-9/*')).toEqual({ start: 5, end: 9 })+ })++ it.each(malformedRanges)('rejects malformed range "%s"', (value) => {+ expect(() => parseContentRange(value)).toThrow(new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed'))+ })+})diff --git a/backend/src/applications/files/utils/upload-file.ts b/backend/src/applications/files/utils/upload-file.tsindex 199a0eb0..daa1b577 100644--- a/backend/src/applications/files/utils/upload-file.ts+++ b/backend/src/applications/files/utils/upload-file.ts@@ -1,14 +1,132 @@+import { HttpStatus } from '@nestjs/common'import path from 'node:path'import { randomUUID } from 'node:crypto'-import { fileName } from './files'+import { FileError } from '../models/file-error'+import { maxFileSizeExceededError, storageQuotaExceededError } from './errors'+import { temporaryPathPrefix } from './files'const FASTIFY_MULTIPART_FILE_TOO_LARGE_CODE = 'FST_REQ_FILE_TOO_LARGE' as const+export interface UploadFileStreamLimiter {+ readonly initialFileSize: number+ // Checks the number of bytes the current stream is expected to append.+ assertKnownSize: (bytes: number) => void+ // Checks the expected final file size, including the current range offset.+ assertFinalSize: (bytes: number) => void+ consume: (bytes: number) => void+}++export interface UploadQuotaSnapshot {+ storageQuota?: number | null+ storageUsage?: number+}++/**+ * Request-scoped upload limiter. It deliberately does not coordinate quota+ * reservations between concurrent requests or workers.+ */+export class UploadStreamLimiter {+ // Shared by all file limiters created for the same request, notably multipart uploads.+ private quotaConsumed = 0++ constructor(+ private readonly maxFileSize: number,+ private readonly quotaRemaining?: number+ ) {}++ createFileLimiter(initialFileSize = 0): UploadFileStreamLimiter {+ if (!Number.isSafeInteger(initialFileSize) || initialFileSize < 0) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')+ }+ if (initialFileSize > this.maxFileSize) {+ throw maxFileSizeExceededError()+ }++ // A resumed offset counts toward maxFileSize but is not charged again against the quota snapshot.+ let fileSize = initialFileSize+ const assertCanConsume = (bytes: number, consume: boolean) => {+ if (!Number.isSafeInteger(bytes) || bytes < 0) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')+ }++ const fileRemaining = this.maxFileSize - fileSize+ const quotaRemaining = this.quotaRemaining === undefined ? Number.POSITIVE_INFINITY : this.quotaRemaining - this.quotaConsumed++ if (bytes > fileRemaining || bytes > quotaRemaining) {+ // Report the first effective boundary. If both are equal, the file-size+ // limit is the more specific upload error.+ if (quotaRemaining < fileRemaining) {+ throw storageQuotaExceededError()+ }+ throw maxFileSizeExceededError()+ }++ if (consume) {+ fileSize += bytes+ this.quotaConsumed += bytes+ }+ }++ return {+ initialFileSize,+ assertKnownSize: (bytes) => assertCanConsume(bytes, false),+ assertFinalSize: (bytes) => {+ if (!Number.isSafeInteger(bytes) || bytes < fileSize) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid upload size')+ }+ // Convert the declared final size into the number of bytes still expected on this stream.+ assertCanConsume(bytes - fileSize, false)+ },+ consume: (bytes) => assertCanConsume(bytes, true)+ }+ }+}++export function createUploadStreamLimiter(space: UploadQuotaSnapshot | undefined, maxFileSize: number): UploadStreamLimiter {+ // A null quota means unlimited storage; a negative remainder is clamped so every positive write is rejected.+ const quotaRemaining = space?.storageQuota == null ? undefined : Math.max(0, space.storageQuota - (space.storageUsage ?? 0))+ return new UploadStreamLimiter(maxFileSize, quotaRemaining)+}++export function parseContentLength(value: unknown): number | undefined {+ if (value === undefined || value === null) return undefined++ const normalized = typeof value === 'number' ? String(value) : typeof value === 'string' ? value.trim() : ''+ if (!/^\d+$/.test(normalized)) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header')+ }++ const contentLength = Number(normalized)+ if (!Number.isSafeInteger(contentLength)) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Invalid "content-length" header')+ }+ return contentLength+}++export function parseContentRange(value: string): { start: number; end: number; total?: number } {+ const match = /^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i.exec(value)+ if (!match) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')+ }+ const start = Number(match[1])+ const end = Number(match[2])+ const total = match[3] === '*' ? undefined : Number(match[3])+ if (+ !Number.isSafeInteger(start) ||+ !Number.isSafeInteger(end) ||+ end < start ||+ (total !== undefined && (!Number.isSafeInteger(total) || total <= end))+ ) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Content-range : header is malformed')+ }+ return { start, end, ...(total !== undefined && { total }) }+}+export function isMultipartFileTooLargeError(e: any): boolean {// Other multipart limits also return 413; only this code means the file-size limit was reached.return e?.code === FASTIFY_MULTIPART_FILE_TOO_LARGE_CODE}export function uploadTmpFilePath(tmpPath: string, partFileName: string): string {- return path.join(tmpPath, `${randomUUID()}-upload-${fileName(partFileName) || 'file'}`)+ return path.join(tmpPath, `${temporaryPathPrefix(partFileName, 'upload')}${randomUUID()}`)}diff --git a/backend/src/applications/spaces/guards/space.guard.spec.ts b/backend/src/applications/spaces/guards/space.guard.spec.tsindex 1ead234d..7b4b305f 100644--- a/backend/src/applications/spaces/guards/space.guard.spec.ts+++ b/backend/src/applications/spaces/guards/space.guard.spec.ts@@ -350,16 +350,17 @@ describe(SpaceGuard.name, () => {await expect(spacesGuard.canActivate(context)).rejects.toEqual(expect.objectContaining({ status: HttpStatus.INSUFFICIENT_STORAGE }))})- it('should fail if request content-length header exceed the quota', async () => {+ it('should leave generic content-length quota enforcement to the upload consumer', async () => {userTest.role = USER_ROLE.USERconst storageQuota = 12const storageUsage = 11+ const willExceedQuota = vi.fn((contentLength: number) => contentLength > storageQuota - storageUsage)spacesManager.spaceEnv = vi.fn().mockReturnValue({enabled: true,envPermissions: `${SPACE_OPERATION.ADD}`,storageQuota: storageQuota,storageUsage: storageUsage,- willExceedQuota: (contentLength: number) => contentLength > storageQuota - storageUsage+ willExceedQuota} as Partial<SpaceEnv>)context.switchToHttp().getRequest.mockReturnValueOnce({method: 'POST',@@ -367,14 +368,8 @@ describe(SpaceGuard.name, () => {user: userTest,params: { '*': 'files/personal' }})- await expect(spacesGuard.canActivate(context)).rejects.toEqual(expect.objectContaining({ status: HttpStatus.INSUFFICIENT_STORAGE }))- context.switchToHttp().getRequest.mockReturnValueOnce({- method: 'POST',- user: userTest,- headers: {},- params: { '*': 'files/personal' }- })expect(await spacesGuard.canActivate(context)).toBe(true)+ expect(willExceedQuota).not.toHaveBeenCalled()})it('should fail for add and modify operations in trash repository', async () => {diff --git a/backend/src/applications/spaces/guards/space.guard.ts b/backend/src/applications/spaces/guards/space.guard.tsindex 8e4b50bd..27a9de5e 100644--- a/backend/src/applications/spaces/guards/space.guard.ts+++ b/backend/src/applications/spaces/guards/space.guard.ts@@ -49,11 +49,6 @@ export class SpaceGuard implements CanActivate {if (req.space.quotaIsExceeded) {logger.warn(`Storage quota exceeded for *${req.space.alias}* (${req.space.id})`)throw new HttpException(FILE_ERROR.STORAGE_QUOTA_EXCEEDED, HttpStatus.INSUFFICIENT_STORAGE)- } else if (req.space.storageQuota) {- const contentLength = parseInt(req.headers['content-length'] || '0', 10) || 0- if (req.space.willExceedQuota(contentLength)) {- throw new HttpException(FILE_ERROR.STORAGE_QUOTA_EXCEEDED, HttpStatus.INSUFFICIENT_STORAGE)- }}}}diff --git a/backend/src/applications/sync/dtos/sync-upload.dto.ts b/backend/src/applications/sync/dtos/sync-upload.dto.tsindex f3e54fe4..9bdc2ce7 100644--- a/backend/src/applications/sync/dtos/sync-upload.dto.ts+++ b/backend/src/applications/sync/dtos/sync-upload.dto.ts@@ -1,5 +1,5 @@import { Transform } from 'class-transformer'-import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString } from 'class-validator'+import { IsDefined, IsInt, IsNotEmpty, IsOptional, IsString, Min } from 'class-validator'export class SyncUploadDto {@IsOptional()@@ -17,5 +17,6 @@ export class SyncUploadDto {@IsNotEmpty()@Transform(({ value }) => parseInt(value))@IsInt()+ @Min(0)size: number}diff --git a/backend/src/applications/sync/services/sync-manager.service.spec.ts b/backend/src/applications/sync/services/sync-manager.service.spec.tsindex ac0ad836..2e3db7a2 100644--- a/backend/src/applications/sync/services/sync-manager.service.spec.ts+++ b/backend/src/applications/sync/services/sync-manager.service.spec.ts@@ -165,6 +165,7 @@ describe(SyncManager.name, () => {const r = await service.upload(req as any, dto as any)expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {tmpPath: expect.any(String),+ expectedUploadSize: 10,checksumAlg: SYNC_CHECKSUM_ALG,validateTmpFile: expect.any(Function)})@@ -195,6 +196,7 @@ describe(SyncManager.name, () => {const r = await service.upload(req as any, dto as any)expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {tmpPath: expect.any(String),+ expectedUploadSize: 5,validateTmpFile: expect.any(Function)})expect(touchFile).toHaveBeenCalledWith('/tmp/up2.bin', 1710000100)@@ -214,9 +216,10 @@ describe(SyncManager.name, () => {expect(touchFile).not.toHaveBeenCalled()})- it('should reject when tmp file exceeds quota and preserve tmp', async () => {+ it('should delegate the declared size to the streaming limiter without a post-write quota check', async () => {+ const willExceedQuota = vi.fn().mockReturnValue(true)const req = makeReq({- space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota: vi.fn().mockReturnValue(true) }+ space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota }})const dto = { size: 10, mtime: 1710000100 }filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {@@ -224,10 +227,11 @@ describe(SyncManager.name, () => {})fsPromises.stat.mockResolvedValue({ size: 10, ino: 321, mtime: new Date(1710000100 * 1000) })- await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)- expect(req.space.willExceedQuota).toHaveBeenCalledWith(10)+ await expect(service.upload(req as any, dto as any)).resolves.toEqual({ ino: 321 })+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, expect.objectContaining({ expectedUploadSize: 10 }))+ expect(willExceedQuota).not.toHaveBeenCalled()expect(removeFiles).not.toHaveBeenCalled()- expect(touchFile).not.toHaveBeenCalled()+ expect(touchFile).toHaveBeenCalled()})})@@ -585,7 +589,7 @@ describe(SyncManager.name, () => {await expect((async () => {for await (const _ of iter) {- /* consume */+ // consume}})()).rejects.toThrow('Unable to parse path')diff --git a/backend/src/applications/sync/services/sync-manager.service.ts b/backend/src/applications/sync/services/sync-manager.service.tsindex aa672ee5..a2d5cd2b 100644--- a/backend/src/applications/sync/services/sync-manager.service.ts+++ b/backend/src/applications/sync/services/sync-manager.service.ts@@ -50,15 +50,13 @@ export class SyncManager {try {await this.filesManager.saveStream(req.user, req.space, req, {tmpPath: tmpPath,+ expectedUploadSize: syncUploadDto.size,...(syncUploadDto.checksum && { checksumAlg: SYNC_CHECKSUM_ALG }),validateTmpFile: async ({ tmpPath, checksum }) => {const tmpStats = await fs.stat(tmpPath)if (tmpStats.size !== syncUploadDto.size) {throw new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${tmpStats.size} != ${syncUploadDto.size}`)}- if (req.space.storageQuota && req.space.willExceedQuota(tmpStats.size)) {- throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR.STORAGE_QUOTA_EXCEEDED)- }if (syncUploadDto.checksum && checksum !== syncUploadDto.checksum) {throw new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical')}diff --git a/backend/src/applications/webdav/services/webdav-methods.service.spec.ts b/backend/src/applications/webdav/services/webdav-methods.service.spec.tsindex ecd71f84..20a5d4af 100644--- a/backend/src/applications/webdav/services/webdav-methods.service.spec.ts+++ b/backend/src/applications/webdav/services/webdav-methods.service.spec.ts@@ -6,7 +6,7 @@ import { FileError } from '../../files/models/file-error'import { LockConflict } from '../../files/models/file-lock-error'import { FilesLockManager } from '../../files/services/files-lock-manager.service'import { FilesManager } from '../../files/services/files-manager.service'-import { dirName, genEtag, isPathExists } from '../../files/utils/files'+import { dirName, genEtag, isPathExists, removeFiles } from '../../files/utils/files'import { SPACE_REPOSITORY } from '../../spaces/constants/spaces'import * as PathsUtils from '../../spaces/utils/paths'import { haveSpaceEnvPermissions } from '../../spaces/utils/permissions'@@ -22,7 +22,10 @@ vi.mock('../../files/utils/files', () => ({isPathIsDir: vi.fn(),fileName: vi.fn().mockReturnValue('fileName'),dirName: vi.fn(),- genEtag: vi.fn().mockReturnValue('W/"etag-123"')+ genEtag: vi.fn().mockReturnValue('W/"etag-123"'),+ removeFiles: vi.fn(() => Promise.resolve()),+ tempFilePath: vi.fn().mockReturnValue('/tmp/test-user/file.txt-upload-id'),+ temporaryPathPrefix: vi.fn().mockReturnValue('file.txt-upload-')}))vi.mock('../../spaces/utils/permissions', () => ({@@ -75,7 +78,8 @@ describe('WebDAVMethods', () => {const createBaseRequest = (overrides: Partial<any> = {}) =>({method: 'GET',- user: { id: 1, login: 'test-user', fullName: 'Test User', email: '[email protected]' },+ headers: {},+ user: { id: 1, login: 'test-user', fullName: 'Test User', email: '[email protected]', tmpPath: '/tmp/test-user' },dav: {url: '/webdav/test/file.txt',depth: '0',@@ -631,7 +635,12 @@ describe('WebDAVMethods', () => {const result = await service.put(req, res)- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, expect.objectContaining({ dav: expect.any(Object) }))+ expect(filesManager.saveStream).toHaveBeenCalledWith(+ req.user,+ req.space,+ req,+ expect.objectContaining({ dav: expect.any(Object), tmpPath: '/tmp/test-user/file.txt-upload-id' })+ )expect(res.statusCode).toBe(HttpStatus.NO_CONTENT)expect(res.headers['etag']).toBeDefined()expect(result).toBe(res)@@ -641,9 +650,11 @@ describe('WebDAVMethods', () => {filesManager.saveStream.mockResolvedValue(false) // file didn't existconst req = createBaseRequest({ method: 'PUT' })const res = createMockResponse()+ vi.mocked(isPathExists).mockImplementation(async (rPath: string) => rPath === '/real/path/to')const result = await service.put(req, res)+ expect(filesManager.saveStream.mock.calls[0][3]).not.toHaveProperty('tmpPath')expect(res.statusCode).toBe(HttpStatus.CREATED)expect(res.headers['etag']).toBeDefined()expect(result).toBe(res)@@ -674,6 +685,16 @@ describe('WebDAVMethods', () => {}))})++ it('should write content-range updates directly to preserve resumability', async () => {+ filesManager.saveStream.mockResolvedValue(true)+ const req = createBaseRequest({ method: 'PUT', headers: { 'content-range': 'bytes 4-7/8' } })+ const res = createMockResponse()++ await service.put(req, res)++ expect(filesManager.saveStream.mock.calls[0][3]).not.toHaveProperty('tmpPath')+ })})describe('Error handling', () => {@@ -704,6 +725,17 @@ describe('WebDAVMethods', () => {expect(res.statusCode).toBe(409)expect(res.body).toBe('File conflict')+ expect(removeFiles).toHaveBeenCalledWith('/tmp/test-user/file.txt-upload-id')+ })++ it('should not clean up direct content-range updates after an error', async () => {+ filesManager.saveStream.mockRejectedValue(new FileError(HttpStatus.BAD_REQUEST, 'Interrupted upload'))+ const req = createBaseRequest({ method: 'PUT', headers: { 'content-range': 'bytes 4-7/8' } })+ const res = createMockResponse()++ await service.put(req, res)++ expect(removeFiles).not.toHaveBeenCalled()})it('should throw HttpException for unexpected errors', async () => {@@ -1524,7 +1556,7 @@ describe('WebDAVMethods', () => {describe('haveLock condition', () => {it('should return true when haveLock matches (lock exists, mustMatch=true)', async () => {- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockResolvedValue([{ key: 'lock1' }] as any)const req = createBaseRequest({dav: {@@ -1540,7 +1572,7 @@ describe('WebDAVMethods', () => {})it('should return true when haveLock matches (no lock, mustMatch=false)', async () => {- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockResolvedValue([])const req = createBaseRequest({dav: {@@ -1556,7 +1588,7 @@ describe('WebDAVMethods', () => {})it('should return false with 412 when haveLock mismatches (lock exists, mustMatch=false)', async () => {- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockResolvedValue([{ key: 'lock1' }] as any)const req = createBaseRequest({dav: {@@ -1573,7 +1605,7 @@ describe('WebDAVMethods', () => {})it('should return false with 412 when haveLock mismatches (no lock, mustMatch=true)', async () => {- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockResolvedValue([])const req = createBaseRequest({dav: {@@ -1590,7 +1622,7 @@ describe('WebDAVMethods', () => {})it('should return false with 412 when haveLock lookup throws error', async () => {- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockRejectedValue(new Error('Database error'))const req = createBaseRequest({dav: {@@ -1797,7 +1829,7 @@ describe('WebDAVMethods', () => {describe('Multiple conditions', () => {it('should evaluate multiple conditions and return true if any matches', async () => {vi.mocked(isPathExists).mockResolvedValue(true)- vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1 })+ vi.mocked(PathsUtils.dbFileFromSpace).mockReturnValue({ path: 'file.txt', spaceId: 1, inTrash: false })filesLockManager.getLocksByPath.mockResolvedValue([])const req = createBaseRequest({dav: {diff --git a/backend/src/applications/webdav/services/webdav-methods.service.ts b/backend/src/applications/webdav/services/webdav-methods.service.tsindex a8e8ae21..35982d1c 100644--- a/backend/src/applications/webdav/services/webdav-methods.service.ts+++ b/backend/src/applications/webdav/services/webdav-methods.service.ts@@ -7,7 +7,7 @@ import { FileError } from '../../files/models/file-error'import { LockConflict } from '../../files/models/file-lock-error'import { FilesLockManager } from '../../files/services/files-lock-manager.service'import { FilesManager } from '../../files/services/files-manager.service'-import { dirName, fileName, genEtag, isPathExists, isPathIsDir } from '../../files/utils/files'+import { dirName, fileName, genEtag, isPathExists, isPathIsDir, removeFiles, tempFilePath, temporaryPathPrefix } from '../../files/utils/files'import { SPACE_OPERATION, SPACE_REPOSITORY } from '../../spaces/constants/spaces'import { SpaceEnv } from '../../spaces/models/space-env.model'import { dbFileFromSpace } from '../../spaces/utils/paths'@@ -216,15 +216,33 @@ export class WebDAVMethods {@IfHeaderDecorator()async put(req: FastifyDAVRequest, res: FastifyReply): Promise<FastifyReply> {+ const isRangeUpload = req.headers['content-range'] !== undefined+ let tmpPath: string | undefinedlet rExists: booleantry {+ // saveStream creates the parent of a supplied tmpPath, so keep the WebDAV destination+ // parent check here to preserve the expected 409 response.+ if (!(await isPathExists(dirName(req.space.realPath)))) {+ throw new FileError(HttpStatus.CONFLICT, 'Parent must exists')+ }+ // Stage only complete overwrites to preserve the previous file on failure. New files and+ // ranged uploads write directly because their partial content is the checkpoint used to resume.+ if (!isRangeUpload && (await isPathExists(req.space.realPath))) {+ tmpPath = tempFilePath(req.user.tmpPath, temporaryPathPrefix(req.space.realPath, 'upload'))+ }rExists = await this.filesManager.saveStream(req.user, req.space, req, {+ ...(tmpPath && { tmpPath }),dav: {depth: req.dav.depth,lockTokens: extractAllTokens(req.dav.ifHeaders)}})} catch (e) {+ if (tmpPath) {+ await removeFiles(tmpPath).catch((cleanupError: Error) =>+ this.logger.error({ tag: this.put.name, msg: `Unable to remove WebDAV staging file ${tmpPath}: ${cleanupError}` })+ )+ }return this.handleError(req, res, e)}return resdiff --git a/backend/src/applications/webdav/utils/bootstrap.ts b/backend/src/applications/webdav/utils/bootstrap.tsindex c9e1e2a6..ed0818fa 100644--- a/backend/src/applications/webdav/utils/bootstrap.ts+++ b/backend/src/applications/webdav/utils/bootstrap.ts@@ -29,6 +29,8 @@ export function bootstrapWebDAV(app: NestFastifyApplication, fastifyInstance: Fa* WebDAV clients may send JSON/XML files with a Content-Type that triggers* Fastify's body parsers (application/json, text/plain, application/xml...).* For PUT uploads we must always treat the payload as a raw binary stream.+ * Sync uploads are intentionally not handled here: the Sync client contract+ * requires application/octet-stream, while other Sync POST routes use parsed bodies.** This hook forces `application/octet-stream` for WebDAV PUT requests only,* leaving XML-based WebDAV methods unaffected.diff --git a/environment/environment.dist.yaml b/environment/environment.dist.yamlindex f1156719..1b28f7e1 100755--- a/environment/environment.dist.yaml+++ b/environment/environment.dist.yaml@@ -331,7 +331,7 @@ applications:files:# requireddataPath: /home/sync-in- # maxUploadSize: Maximum upload file size.+ # maxUploadSize: Maximum size in bytes for a file received through uploads, WebDAV, sync, remote downloads, or editor callbacks.# default: 5368709120 (5 GB)maxUploadSize: 5368709120contentIndexing: