Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): stage downloads and archives in user tmp paths before publishing
johaven1363899d1a90
diff --git a/backend/src/applications/files/interfaces/download-file.interface.ts b/backend/src/applications/files/interfaces/download-file.interface.tsindex e9ef3204..66450bf5 100644--- a/backend/src/applications/files/interfaces/download-file.interface.ts+++ b/backend/src/applications/files/interfaces/download-file.interface.ts@@ -9,6 +9,7 @@ export interface DownloadFileContentInfo {export interface DownloadFileOptions {allowPrivateIP?: booleanspace?: SpaceEnv+ publishedPath?: stringgetContentInfo?: booleanmaxSize?: number}diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex c5c48e22..858dd74c 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -95,12 +95,17 @@ describe(FilesManager.name, () => {directory: jest.Mockfile: jest.Mockfinalize: jest.Mock+ abort: jest.Mock}archive.directory = jest.fn().mockReturnValue(archive)archive.file = jest.fn().mockReturnValue(archive)archive.finalize = jest.fn().mockImplementation(async () => {archive.end()})+ archive.abort = jest.fn().mockImplementation(() => {+ archive.end()+ return archive+ });(archiver as unknown as jest.Mock).mockReturnValueOnce(archive as any)jest.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any)return archive@@ -165,6 +170,7 @@ describe(FilesManager.name, () => {jest.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)jest.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)jest.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')+ jest.spyOn(filesUtils, 'tempFilePath').mockReturnValue('/tmp/staged-file')jest.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)jest.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')jest.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)@@ -880,6 +886,7 @@ describe(FilesManager.name, () => {it('should handle HEAD+GET and emit task watch/event', async () => {const space = makeSpace({ task: { cacheKey: 'task-1', props: {} } });(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')http.axiosRef.mockResolvedValueOnce({headers: { 'content-length': '55' },@@ -895,8 +902,21 @@ describe(FilesManager.name, () => {await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })expect(space.task.props.totalSize).toBe(55)- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DOWNLOAD, '/tmp/download.txt')- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/download.txt', expect.anything(), 0, 55)+ expect(taskEmitSpy).toHaveBeenCalledWith(+ 'startWatch',+ space,+ FILE_OPERATION.DOWNLOAD,+ '/tmp/download.txt',+ '/data/users/john/tmp/download.txt-download-uuid'+ )+ expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', expect.anything(), 0, 55)+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', '/tmp/download.txt')+ expect(filesLockManager.create).toHaveBeenCalledWith(+ user,+ expect.objectContaining({ path: 'download.txt' }),+ expect.any(String),+ DEPTH.RESOURCE+ )expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')expect(fileEmitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })})@@ -905,6 +925,7 @@ describe(FilesManager.name, () => {const error = new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR_MESSAGES.MAX_FILE_SIZE_EXCEEDED)const space = makeSpace();(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid');(filesUtils.writeFromStream as jest.Mock).mockRejectedValueOnce(error)http.axiosRef.mockResolvedValueOnce({@@ -919,7 +940,32 @@ describe(FilesManager.name, () => {await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBe(error)- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/download.txt')+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid')+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')+ expect(fileEmitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })+ })++ it('should cleanup temporary file and skip ADD event when publishing download fails', async () => {+ const error = new Error('move failed')+ const space = makeSpace()+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)+ http.axiosRef+ .mockResolvedValueOnce({+ headers: { 'content-length': '55' },+ request: { socket: { remoteAddress: '8.8.8.8' } }+ })+ .mockResolvedValueOnce({+ data: Readable.from(['abc']),+ request: { socket: { remoteAddress: '8.8.8.8' } }+ })+ const fileEmitSpy = jest.spyOn(FileEvent, 'emit')++ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBe(error)++ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid')expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')expect(fileEmitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })})@@ -929,6 +975,7 @@ describe(FilesManager.name, () => {it('should archive files and emit events', async () => {const archive = createArchiveMock();(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid');(filesUtils.isPathIsDir as jest.Mock).mockImplementation(async (p: string) => p.endsWith('/dir'))const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })const dto = {@@ -948,12 +995,21 @@ describe(FilesManager.name, () => {expect(archive.directory).toHaveBeenCalled()expect(archive.file).toHaveBeenCalled()expect(archive.finalize).toHaveBeenCalled()- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.COMPRESS, '/tmp/archive.tar.gz')+ expect(taskEmitSpy).toHaveBeenCalledWith(+ 'startWatch',+ space,+ FILE_OPERATION.COMPRESS,+ '/tmp/archive.tar.gz',+ '/data/users/john/tmp/archive.tar.gz-compress-uuid'+ )+ expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', { highWaterMark: expect.any(Number) })+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', '/tmp/archive.tar.gz')})it('should allow archive export from trash when compressInDirectory is false', async () => {const archive = createArchiveMock();(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive-trash.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid');(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)const emitSpy = jest.spyOn(FileEvent, 'emit')const space = makeTrashSpace({@@ -971,8 +1027,121 @@ describe(FilesManager.name, () => {await expect(service.compress(user, space, dto)).resolves.toBeUndefined()expect(filesLockManager.create).not.toHaveBeenCalled()expect(archive.file).toHaveBeenCalled()+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid', '/tmp/archive-trash.tar.gz')expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tar.gz' })})++ it('should cleanup temporary archive and skip ADD event when publishing archive fails', async () => {+ const archive = createArchiveMock()+ const error = new Error('move failed')+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ ;(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)+ const emitSpy = jest.spyOn(FileEvent, 'emit')+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })+ const dto = {+ name: 'archive',+ extension: 'tar.gz',+ compressInDirectory: false,+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]+ } as any++ await expect(service.compress(user, space, dto)).rejects.toBe(error)++ expect(archive.abort).toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan((filesUtils.removeFiles as jest.Mock).mock.invocationCallOrder[0])+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })+ })++ it('should abort archive pipeline and cleanup temporary file when preparing an entry fails', async () => {+ const archive = createArchiveMock()+ const error = new Error('unable to prepare entry')+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ ;(filesUtils.isPathIsDir as jest.Mock).mockRejectedValueOnce(error)+ const emitSpy = jest.spyOn(FileEvent, 'emit')+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })+ const dto = {+ name: 'archive',+ extension: 'tar.gz',+ compressInDirectory: false,+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]+ } as any++ await expect(service.compress(user, space, dto)).rejects.toBe(error)++ expect(archive.abort).toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan((filesUtils.removeFiles as jest.Mock).mock.invocationCallOrder[0])+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })+ })++ it('should cleanup temporary archive without waiting for finalize after pipeline failure', async () => {+ const archive = createArchiveMock()+ const error = new Error('pipeline failed')+ archive.finalize.mockImplementationOnce(() => {+ archive.emit('error', error)+ return new Promise<void>(() => undefined)+ })+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ ;(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)+ const emitSpy = jest.spyOn(FileEvent, 'emit')+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })+ const dto = {+ name: 'archive',+ extension: 'tar.gz',+ compressInDirectory: false,+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]+ } as any++ await expect(service.compress(user, space, dto)).rejects.toBe(error)++ expect(archive.abort).toHaveBeenCalled()+ expect(archive.destroyed).toBe(true)+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })+ })++ it('should not append entries after archive pipeline cancellation', async () => {+ const archive = createArchiveMock()+ const error = new Error('pipeline failed')+ let resolveEntry!: (isDir: boolean) => void+ let entryStarted!: () => void+ const entryStartedPromise = new Promise<void>((resolve) => {+ entryStarted = resolve+ })+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ ;(filesUtils.isPathIsDir as jest.Mock).mockImplementationOnce(+ () =>+ new Promise<boolean>((resolve) => {+ resolveEntry = resolve+ entryStarted()+ })+ )+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })+ const dto = {+ name: 'archive',+ extension: 'tar.gz',+ compressInDirectory: false,+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]+ } as any++ const compressPromise = service.compress(user, space, dto)+ await entryStartedPromise+ archive.emit('error', error)+ await new Promise<void>((resolve) => setImmediate(resolve))++ expect(filesUtils.removeFiles).not.toHaveBeenCalled()+ resolveEntry(false)+ await expect(compressPromise).rejects.toBe(error)+ expect(archive.file).not.toHaveBeenCalled()+ expect(archive.finalize).not.toHaveBeenCalled()+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ })})describe('decompress', () => {diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 0d4bf73b..5f3f691e 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -48,6 +48,7 @@ import {makeTempDir,moveFiles,removeFiles,+ tempFilePath,touchFile,uniqueDatedFilePath,uniqueFilePathFromDir,@@ -569,9 +570,10 @@ export class FilesManager {async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {this.checkNotTrashRepository(space)this.logger.log({ tag: this.downloadFromUrl.name, msg: `${downloadDto.url}` })- const rPath = await uniqueFilePathFromDir(space.realPath)+ const dstPath = await uniqueFilePathFromDir(space.realPath)+ const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)const dbFile = space.dbFile- dbFile.path = path.join(dirName(dbFile.path), fileName(space.realPath))+ dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))// create lockconst [ok, fileLock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.RESOURCE)@@ -580,15 +582,18 @@ export class FilesManager {}try {- await new DownloadFile(this.http).download(downloadDto, rPath, { space: space })+ await new DownloadFile(this.http).download(downloadDto, tmpPath, { space: space, publishedPath: dstPath })+ await moveFiles(tmpPath, dstPath)} catch (e) {- await removeFiles(rPath).catch((err: Error) => this.logger.error({ tag: this.downloadFromUrl.name, msg: `unable to remove ${rPath} : ${err}` }))+ await removeFiles(tmpPath).catch((err: Error) =>+ this.logger.error({ tag: this.downloadFromUrl.name, msg: `unable to remove ${tmpPath} : ${err}` })+ )throw e} finally {// release lockawait this.filesLockManager.removeLock(fileLock.key)}- FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: rPath })+ FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })}async compress(user: UserModel, space: SpaceEnv, dto: CompressFileDto): Promise<void> {@@ -600,6 +605,7 @@ export class FilesManager {const srcPath = dirName(space.realPath)const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))+ const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)// avoid using ZIP here because it can trigger high memory usage.const archive: Archiver = archiver(TAR_EXTENSION, {gzip: dto.extension === TAR_GZ_EXTENSION,@@ -620,30 +626,46 @@ export class FilesManager {}if (space.task?.cacheKey) {space.task.props.compressInDirectory = dto.compressInDirectory- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath)+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)}// do+ let aborted = false+ let pipePromise: Promise<void> | undefined+ let entriesPromise: Promise<void> | undefinedtry {- const dstStream = fs.createWriteStream(dstPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- const pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream-- for (const f of dto.files) {- if (await isPathIsDir(f.path)) {- archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)- } else {- archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })+ const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })+ pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream+ entriesPromise = (async () => {+ for (const f of dto.files) {+ const isDir = await isPathIsDir(f.path)+ if (aborted) return+ if (isDir) {+ archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)+ } else {+ archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })+ }}- }-- const finalizePromise = archive.finalize()- await Promise.all([finalizePromise, pipePromise])+ // The pipeline is the completion signal: finalize() may remain pending after abort().+ if (!aborted) {+ void archive.finalize().catch(() => undefined)+ }+ })()+ await Promise.all([entriesPromise, pipePromise])+ await moveFiles(tmpPath, dstPath)+ } catch (e) {+ aborted = true+ archive.abort()+ archive.destroy()+ await Promise.allSettled([entriesPromise, pipePromise].filter((promise): promise is Promise<void> => !!promise))+ await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.compress.name, msg: `unable to remove ${tmpPath} : ${err}` }))+ throw e} finally {if (fileLock) {await this.filesLockManager.removeLock(fileLock.key)}- // emit file event- FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })}+ // emit file event+ FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })}async decompress(user: UserModel, space: SpaceEnv): Promise<void> {diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.tsindex e8c86611..36de13e7 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts@@ -86,7 +86,95 @@ describe(FilesTasksManager.name, () => {FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')- expect(startWatchSpy).toHaveBeenCalledWith(space, FILE_OPERATION.DECOMPRESS, '/tmp/archive-extract', 'files/personal', 'archive')+ expect(startWatchSpy).toHaveBeenCalledWith(+ space,+ FILE_OPERATION.DECOMPRESS,+ '/tmp/archive-extract',+ 'files/personal',+ 'archive',+ '/files/archive'+ )+ })++ it('should watch the published archive after the temporary archive has been moved', async () => {+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const space = { task: { cacheKey: 'task-1', props: {} } } as any++ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')++ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')+ expect(fileSizeSpy).toHaveBeenCalledWith('/files/archive.tgz')+ expect(space.task.props.size).toBe(42)+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })+ })++ it('should retry the published archive when the temporary archive is moved before reading it', async () => {+ const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const space = { task: { cacheKey: 'task-1', props: {} } } as any++ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')++ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')+ expect(fileSizeSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')+ expect(fileSizeSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')+ expect(space.task.props.size).toBe(42)+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })+ })++ it('should retry the published directory when the temporary directory is moved while reading it', async () => {+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)+ const countDirEntriesSpy = jest+ .spyOn(filesUtils, 'countDirEntries')+ .mockResolvedValueOnce({ files: 0, directories: 0 })+ .mockResolvedValueOnce({ files: 2, directories: 1 })+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const space = { task: { cacheKey: 'task-1', props: {} } } as any++ await (filesTasksManager as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')++ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')+ expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')+ expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')+ expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')+ expect(space.task.props).toEqual({ files: 2, directories: 1 })+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1 })+ })++ it('should silently skip watcher update while temporary and published paths are missing', async () => {+ jest.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize')+ const loggerErrorSpy = jest.spyOn((filesTasksManager as any).logger, 'error')+ const stopWatchSpy = jest.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const space = { task: { cacheKey: 'task-1', props: {} } } as any++ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')++ expect(loggerErrorSpy).not.toHaveBeenCalled()+ expect(stopWatchSpy).not.toHaveBeenCalled()+ expect(updateTaskSpy).not.toHaveBeenCalled()+ expect(fileSizeSpy).not.toHaveBeenCalled()+ })++ it('should stop a watcher without staging when its path is missing', async () => {+ const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })+ jest.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)+ const loggerErrorSpy = jest.spyOn((filesTasksManager as any).logger, 'error')+ const stopWatchSpy = jest.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)+ const space = { task: { cacheKey: 'task-1', props: {} } } as any++ await (filesTasksManager as any).updateCompressTask(space, '/files/archive.tgz')++ expect(loggerErrorSpy).toHaveBeenCalled()+ expect(stopWatchSpy).toHaveBeenCalledWith('task-1')})it('should create a task and mark it as success when the async method resolves', async () => {diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.tsindex bb03875b..53a7cdc2 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts@@ -11,7 +11,7 @@ import { UserModel } from '../../users/models/user.model'import { CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'import { FILE_OPERATION } from '../constants/operations'import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'-import { countDirEntries, dirName, fileName, fileSize, isPathIsDir, removeFiles } from '../utils/files'+import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'import { SendFile } from '../utils/send-file'import { FilesMethods } from './files-methods.service'import { FileTaskEvent } from '../events/file-events'@@ -28,7 +28,7 @@ export class FilesTasksManager {private readonly filesMethods: FilesMethods) {FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string) =>- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath))+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined))}@@ -144,7 +144,14 @@ export class FilesTasksManager {}}- private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string, taskName = fileName(rPath)): Promise<void> {+ private async startWatch(+ space: SpaceEnv,+ taskType: FILE_OPERATION,+ rPath: string,+ taskPath?: string,+ taskName = fileName(rPath),+ publishedPath?: string+ ): Promise<void> {if (!space.task?.cacheKey || space.task.cacheKey in this.tasksWatcher) returnthis.logger.verbose({ tag: this.startWatch.name, msg: `${space.task.cacheKey}` })this.updateTask(space.task.cacheKey, space.task?.props, {@@ -153,13 +160,13 @@ export class FilesTasksManager {}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))switch (taskType) {case FILE_OPERATION.COMPRESS:- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateCompressTask(space, rPath), this.watchInterval)+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateCompressTask(space, rPath, publishedPath), this.watchInterval)returncase FILE_OPERATION.DECOMPRESS:- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDecompressTask(space, rPath), this.watchInterval)+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDecompressTask(space, rPath, publishedPath), this.watchInterval)returncase FILE_OPERATION.DOWNLOAD:- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDownloadTask(space, rPath), this.watchInterval)+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDownloadTask(space, rPath, publishedPath), this.watchInterval)returncase FILE_OPERATION.COPY:case FILE_OPERATION.MOVE:@@ -178,9 +185,11 @@ export class FilesTasksManager {delete this.tasksWatcher[cacheKey]}- private async updateCompressTask(space: SpaceEnv, rPath: string): Promise<void> {+ private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {try {- space.task.props.size = await fileSize(rPath)+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)+ if (size === undefined) return+ space.task.props.size = sizeawait this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` }))@@ -190,9 +199,11 @@ export class FilesTasksManager {}}- private async updateDecompressTask(space: SpaceEnv, rPath: string): Promise<void> {+ private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {try {- space.task.props = await countDirEntries(rPath)+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntries)+ if (props === undefined) return+ space.task.props = propsthis.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` }))@@ -202,9 +213,9 @@ export class FilesTasksManager {}}- private async updateDownloadTask(space: SpaceEnv, rPath: string): Promise<void> {+ private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {try {- await this.calcSizeAndProgressTask(space, rPath)+ if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) returnthis.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` }))@@ -230,10 +241,28 @@ export class FilesTasksManager {}}- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string) {- space.task.props.size = await fileSize(rPath)+ private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)+ if (size === undefined) return false+ space.task.props.size = sizeif (space.task.props.totalSize) {space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize}+ return true+ }++ private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {+ if (!publishedPath) return read(rPath)+ if (await isPathExists(rPath)) {+ try {+ const value = await read(rPath)+ // The temporary path may be moved atomically while it is being read.+ if (await isPathExists(rPath)) return value+ } catch (e) {+ // Retry the published path only when the temporary path disappeared during publication.+ if (e?.code !== 'ENOENT') throw e+ }+ }+ if (await isPathExists(publishedPath)) return read(publishedPath)}}diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.tsindex 77212f55..3e2ba08e 100644--- a/backend/src/applications/files/utils/download-file.ts+++ b/backend/src/applications/files/utils/download-file.ts@@ -37,12 +37,12 @@ export class DownloadFile {async download(downloadDto: DownloadFileDto,dstPath: string,- options: { allowPrivateIP?: boolean; space?: SpaceEnv; getContentInfo: true; maxSize?: number }+ options: { allowPrivateIP?: boolean; space?: SpaceEnv; publishedPath?: string; getContentInfo: true; maxSize?: number }): Promise<DownloadFileContentInfo>async download(downloadDto: DownloadFileDto,dstPath: string,- options?: { allowPrivateIP?: boolean; space?: SpaceEnv; getContentInfo?: false | undefined; maxSize?: number }+ options?: { allowPrivateIP?: boolean; space?: SpaceEnv; publishedPath?: string; getContentInfo?: false | undefined; maxSize?: number }): Promise<void>async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions): Promise<void | DownloadFileContentInfo> {const identityEncodingConfig = { decompress: false, headers: { 'Accept-Encoding': 'identity' } }@@ -67,7 +67,7 @@ export class DownloadFile {if (maxSize === null || (contentLength === null && options?.space)) {throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR_MESSAGES.DOWNLOAD_INVALID_CONTENT_LENGTH)}- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, dstPath)+ if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath, dstPath)// The HEAD request resolved redirects; the GET must target that final URL directly.const { response: getRes } = await this.request(@@ -173,14 +173,18 @@ export class DownloadFile {return Number.isSafeInteger(contentLength) ? contentLength : null}- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, dstPath: string): void {+ private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string, dstPath: string): void {if (!space) returnif (space.willExceedQuota(contentLength)) {throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)}if (space.task?.cacheKey) {space.task.props.totalSize = contentLength- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)+ if (publishedPath === dstPath) {+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)+ } else {+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, publishedPath, dstPath)+ }}}}diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.tsindex a9cfe7c8..262f55ea 100644--- a/backend/src/applications/files/utils/files.spec.ts+++ b/backend/src/applications/files/utils/files.spec.ts@@ -5,7 +5,7 @@ import path from 'node:path'import { Readable } from 'node:stream'import { FileError } from '../models/file-error'import { FILE_ERROR_MESSAGES } from './errors'-import { isPathInside, makeTempDir, writeFromStream } from './files'+import { isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'describe(isPathInside.name, () => {const basePath = path.join(path.sep, 'tmp', 'output')@@ -88,3 +88,21 @@ describe(makeTempDir.name, () => {await expect(access(secondPath)).resolves.toBeUndefined()})})++describe(tempFilePath.name, () => {+ it('returns distinct paths with the requested parent and prefix', () => {+ const parentPath = path.join(path.sep, 'tmp', 'user')+ const firstPath = tempFilePath(parentPath, 'archive-compress-')+ const secondPath = tempFilePath(parentPath, 'archive-compress-')++ expect(firstPath).not.toBe(secondPath)+ expect(path.dirname(firstPath)).toBe(parentPath)+ expect(path.basename(firstPath)).toMatch(/^archive-compress-/)+ })++ it('keeps paths with traversal prefixes inside the requested parent', () => {+ const parentPath = path.join(path.sep, 'tmp', 'user')++ expect(path.dirname(tempFilePath(parentPath, path.join('..', 'archive-')))).toBe(parentPath)+ })+})diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex 8f267acd..39e6c791 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -99,6 +99,10 @@ export async function makeTempDir(parentPath: string, prefix: string): Promise<sreturn fs.mkdtemp(path.join(parentPath, prefix))}+export function tempFilePath(parentPath: string, prefix: string): string {+ return path.join(parentPath, `${path.basename(prefix)}${crypto.randomUUID()}`)+}+export function getMimeType(fPath: string, isDir: boolean): string {if (isDir) {return 'directory'