Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): track download, compression and extraction progress
johavencaa6a9285800
diff --git a/backend/src/applications/files/interfaces/download-file.interface.ts b/backend/src/applications/files/interfaces/download-file.interface.tsindex acd0e0c4..7da6962f 100644--- a/backend/src/applications/files/interfaces/download-file.interface.ts+++ b/backend/src/applications/files/interfaces/download-file.interface.ts@@ -12,5 +12,6 @@ export interface DownloadFileOptions {publishedPath?: stringgetContentInfo?: booleanmaxSize?: number+ onProgress?: (bytes: number) => voidsignal?: AbortSignal}diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.tsindex b90bab84..3061c6a4 100644--- a/backend/src/applications/files/interfaces/file-event.interface.ts+++ b/backend/src/applications/files/interfaces/file-event.interface.ts@@ -1,10 +1,9 @@import type { SpaceEnv } from '../../spaces/models/space-env.model'-import type { FILE_OPERATION } from '../constants/operations'import type { ACTION } from '../../../common/constants'import type { UserModel } from '../../users/models/user.model'export interface FileTaskEventEmit {- startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string]+ startWatch: [space: SpaceEnv, rPath: string]}export interface FileEventType {diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.tsindex 82b5efc9..16299c5f 100644--- a/backend/src/applications/files/interfaces/file-task.interface.ts+++ b/backend/src/applications/files/interfaces/file-task.interface.ts@@ -9,7 +9,7 @@ export interface FileTaskTransferOptions {beforeCommit?: () => Promise<void>cacheKey: stringcrossDevice?: boolean- onTransferStart?: (temporaryPath: string) => void+ onTransferStart?: () => voidonProgress?: (bytes: number) => voidoverwrite?: booleansignal: AbortSignal@@ -20,3 +20,9 @@ export interface FileTaskCopyTaskOptions extends FileTaskTransferOptions {preserveTimestamps?: booleanrecursive?: boolean}++export interface FileTaskExtractionEntry {+ path: string+ isDirectory: boolean+ size: number+}diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex a80f6b00..33f8280f 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -15,7 +15,6 @@ import * as spacesPathUtils from '../../spaces/utils/paths'import * as spacesPermsUtils from '../../spaces/utils/permissions'import { DEPTH } from '../../webdav/constants/webdav'import { ACTION } from '../../../common/constants'-import { FILE_OPERATION } from '../constants/operations'import { DownloadFileDto } from '../dto/file-operations.dto'import { FileEvent, FileTaskEvent } from '../events/file-events'import { FileError, SourceCleanupError } from '../models/file-error'@@ -43,7 +42,7 @@ vi.mock('node:dns/promises', () => ({describe(FilesManager.name, () => {let service: FilesManager- let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock }+ let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock; createByteProgressHandler: Mock; createExtractionProgressHandler: Mock }let http: { axiosRef: Mock }const lookupMock = lookup as Mocklet filesQueries: { moveFiles: Mock; deleteFiles: Mock }@@ -172,7 +171,13 @@ describe(FilesManager.name, () => {}),move: vi.fn().mockResolvedValue(undefined),- delete: vi.fn().mockResolvedValue(undefined)+ delete: vi.fn().mockResolvedValue(undefined),+ createByteProgressHandler: vi.fn((space) =>+ vi.fn((bytes: number) => {+ space.task.props.size = (space.task.props.size || 0) + bytes+ })+ ),+ createExtractionProgressHandler: vi.fn().mockReturnValue(vi.fn())}filesLockManager = {create: vi.fn().mockResolvedValue([true, { key: 'lock-1' }]),@@ -1054,17 +1059,19 @@ describe(FilesManager.name, () => {await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })- expect(space.task.props.totalSize).toBe(55)- expect(taskEmitSpy).toHaveBeenCalledWith(- 'startWatch',- space,- FILE_OPERATION.DOWNLOAD,- '/tmp/download.txt',- '/data/users/john/tmp/tasks/.task-1-download.txt'- )+ expect(space.task.props).toMatchObject({ progress: 1, size: 0, totalSize: 55 })+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/download.txt')expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', expect.anything(), 0, 55)+ expect(filesUtils.writeFromStream).toHaveBeenCalledWith(+ '/data/users/john/tmp/tasks/.task-1-download.txt',+ expect.anything(),+ 0,+ 55,+ undefined,+ expect.any(Function)+ )+ expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')expect(filesLockManager.create).toHaveBeenCalledWith(user,@@ -1129,6 +1136,9 @@ describe(FilesManager.name, () => {describe('compress', () => {it('should archive files and emit events', async () => {const archive = createArchiveMock()+ archive.finalize.mockImplementationOnce(async () => {+ archive.end('content')+ })vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })@@ -1149,18 +1159,13 @@ describe(FilesManager.name, () => {expect(archive.directory).toHaveBeenCalled()expect(archive.file).toHaveBeenCalled()expect(archive.finalize).toHaveBeenCalled()- expect(taskEmitSpy).toHaveBeenCalledWith(- 'startWatch',- space,- FILE_OPERATION.COMPRESS,- '/tmp/archive.tar.gz',- '/data/users/john/tmp/tasks/.task-c-archive.tar.gz'- )+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tar.gz')expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {highWaterMark: expect.any(Number)})+ expect(space.task.props.size).toBe(Buffer.byteLength('content'))expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')})@@ -1314,15 +1319,16 @@ describe(FilesManager.name, () => {expect(taskUtils.createTaskTemporaryDir).toHaveBeenCalledWith(user.tasksPath, 'task-d', '/data/users/john/files/archive')expect(filesUtils.makeTempDir).not.toHaveBeenCalled()- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/tasks/.task-d-archive', undefined, undefined)- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')- expect(taskEmitSpy).toHaveBeenCalledWith(- 'startWatch',- space,- FILE_OPERATION.DECOMPRESS,- '/data/users/john/files/archive',- '/data/users/john/tmp/tasks/.task-d-archive'+ expect(filesTasksTransfer.createExtractionProgressHandler).toHaveBeenCalledWith(space)+ expect(unzipSpy).toHaveBeenCalledWith(+ '/data/users/john/files/archive.zip',+ '/data/users/john/tmp/tasks/.task-d-archive',+ undefined,+ undefined,+ expect.any(Function))+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/data/users/john/files/archive')expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')})@@ -1340,6 +1346,7 @@ describe(FilesManager.name, () => {'/data/users/john/tmp/archive-extract-123',true,undefined,+ undefined,undefined)expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')@@ -1354,7 +1361,13 @@ describe(FilesManager.name, () => {await service.decompress(user, space)- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', 60, undefined)+ expect(unzipSpy).toHaveBeenCalledWith(+ '/data/users/john/files/archive.zip',+ '/data/users/john/tmp/archive-extract-123',+ 60,+ undefined,+ undefined+ )})it('should remove partial extraction and skip add event on failure', async () => {diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 8997308e..3f86d130 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -23,7 +23,6 @@ import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'import { CACHE_LOCK_FILE_TTL } from '../constants/cache'import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'-import { FILE_OPERATION } from '../constants/operations'import { ALL_DOCUMENT_TYPES, DEFAULT_DOCUMENT_TYPES, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'import { FileDBProps } from '../interfaces/file-db-props.interface'@@ -36,6 +35,7 @@ import {copyFileContent,copyFiles,createEmptyFile,+ createProgressTransform,dirName,dirSize,fileName,@@ -486,7 +486,7 @@ export class FilesManager {// doif (isMove) {let sourceCleanupError: SourceCleanupError | undefined- if (useTaskTransfer) {+ if (useTaskTransfer && signal) {sourceCleanupError = await this.filesTasksTransfer.move(user, srcSpace, dstSpace, overwrite, isDir, signal, () => this.delete(user, dstSpace))} else {await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)@@ -604,7 +604,12 @@ export class FilesManager {}try {- await new DownloadFile(this.http).download(downloadDto, tmpPath, { space, publishedPath: dstPath, signal })+ await new DownloadFile(this.http).download(downloadDto, tmpPath, {+ space,+ publishedPath: dstPath,+ signal,+ onProgress: isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined+ })signal?.throwIfAborted()await moveFiles(tmpPath, dstPath)} catch (e) {@@ -652,7 +657,8 @@ export class FilesManager {}if (isTaskContext) {space.task!.props.compressInDirectory = dto.compressInDirectory- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)+ space.task!.props.size = 0+ FileTaskEvent.emit('startWatch', space, dstPath)}// dolet aborted = false@@ -660,7 +666,12 @@ export class FilesManager {let entriesPromise: Promise<void> | undefinedtry {const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- pipePromise = pipeline(archive, dstStream, { signal }) // handle archive errors + write stream+ if (isTaskContext) {+ const onProgress = this.filesTasksTransfer.createByteProgressHandler(space)+ pipePromise = pipeline(archive, createProgressTransform(onProgress), dstStream, { signal })+ } else {+ pipePromise = pipeline(archive, dstStream, { signal })+ }entriesPromise = (async () => {for (const f of dto.files) {signal?.throwIfAborted()@@ -723,13 +734,14 @@ export class FilesManager {}fileLock = lock// tasking- if (isTaskContext) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)+ const onEntry = isTaskContext ? this.filesTasksTransfer.createExtractionProgressHandler(space) : undefined+ if (isTaskContext) FileTaskEvent.emit('startWatch', space, dstPath)// doconst maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)if (extension === '.zip') {- await extractZip(space.realPath, tmpPath, maxExtractedSize, signal)+ await extractZip(space.realPath, tmpPath, maxExtractedSize, signal, onEntry)} else {- await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal)+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal, onEntry)}signal?.throwIfAborted()if (await isPathExists(dstPath)) {diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.tsindex becac1ba..30341272 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts@@ -4,7 +4,6 @@ import crypto from 'node:crypto'import { Cache } from '../../../../infrastructure/cache/cache.service'import { SpacesManager } from '../../../spaces/services/spaces-manager.service'import * as filesUtils from '../../utils/files'-import * as tasksUtils from '../../utils/tasks'import { FileTaskEvent } from '../../events/file-events'import { SendFile } from '../../utils/send-file'import { FILE_OPERATION } from '../../constants/operations'@@ -121,9 +120,9 @@ describe(FilesTasksManager.name, () => {it('should unref watchers and clear them on module destroy', async () => {const cacheKey = 'task-1'const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')- const space = { task: { cacheKey, props: {} } } as any+ const space = { url: 'files/personal/document.txt', task: { cacheKey, props: {} } } as anycacheStore.set(cacheKey, { props: {} })- ;(filesTasksWatcher as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')+ ;(filesTasksWatcher as any).startWatch(space, '/files/document.txt')const watcher = (filesTasksWatcher as any).tasksWatcher[cacheKey] as NodeJS.Timeoutexpect(watcher.hasRef()).toBe(false)@@ -218,101 +217,13 @@ describe(FilesTasksManager.name, () => {}})- it('should watch a temporary path while keeping the published file name', () => {+ it('should watch task props while keeping the published file metadata', () => {const startWatchSpy = vi.spyOn(filesTasksWatcher as any, 'startWatch').mockResolvedValueOnce(undefined)const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')+ FileTaskEvent.emit('startWatch', space, '/files/archive')- expect(startWatchSpy).toHaveBeenCalledWith(- space,- FILE_OPERATION.DECOMPRESS,- '/tmp/archive-extract',- 'files/personal',- 'archive',- '/files/archive'- )- })-- it('should watch the published archive after the temporary archive has been moved', async () => {- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)- const space = { task: { cacheKey: 'task-1', props: {} } } as any-- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')-- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')- expect(fileSizeSpy).toHaveBeenCalledWith('/files/archive.tgz')- expect(space.task.props.size).toBe(42)- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })- })-- it('should retry the published archive when the temporary archive is moved before reading it', async () => {- const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)- const space = { task: { cacheKey: 'task-1', props: {} } } as any-- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')-- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')- expect(fileSizeSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')- expect(fileSizeSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')- expect(space.task.props.size).toBe(42)- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })- })-- it('should retry the published directory when the temporary directory is moved while reading it', async () => {- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)- const countDirEntriesSpy = vi- .spyOn(tasksUtils, 'countDirEntriesAndSize')- .mockResolvedValueOnce({ files: 0, directories: 0, size: 0 })- .mockResolvedValueOnce({ files: 2, directories: 1, size: 42 })- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)- const space = { task: { cacheKey: 'task-1', props: {} } } as any-- await (filesTasksWatcher as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')-- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')- expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')- expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')- expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')- expect(space.task.props).toEqual({ files: 2, directories: 1, size: 42 })- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1, size: 42 })- })-- it('should silently skip watcher update while temporary and published paths are missing', async () => {- vi.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize')- const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')- const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)- const space = { task: { cacheKey: 'task-1', props: {} } } as any-- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')-- expect(loggerErrorSpy).not.toHaveBeenCalled()- expect(stopWatchSpy).not.toHaveBeenCalled()- expect(updateTaskSpy).not.toHaveBeenCalled()- expect(fileSizeSpy).not.toHaveBeenCalled()- })-- it('should stop a watcher without staging when its path is missing', async () => {- const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })- vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)- const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')- const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)- const space = { task: { cacheKey: 'task-1', props: {} } } as any-- await (filesTasksWatcher as any).updateCompressTask(space, '/files/archive.tgz')-- expect(loggerErrorSpy).toHaveBeenCalled()- expect(stopWatchSpy).toHaveBeenCalledWith('task-1')+ expect(startWatchSpy).toHaveBeenCalledWith(space, '/files/archive')})it('should not let an in-flight progress update overwrite task completion', async () => {@@ -433,6 +344,31 @@ describe(FilesTasksManager.name, () => {})})+ it('should finalize fast extraction metrics without waiting for a watcher tick', async () => {+ const taskId = '77777777-7777-4777-8777-777777777777'+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)+ filesMethods.doWork.mockImplementationOnce(async (_user, taskSpace) => {+ taskSpace.task.props = { directories: 2, files: 3, size: 120 }+ })+ const user = { id: 7 } as any+ const space = {+ realPath: '/data/users/john/files/archive.zip',+ url: 'files/personal/archive.zip'+ } as any++ await filesTasksManager.createTask(FILE_OPERATION.DECOMPRESS, user, space, null, 'doWork')+ await flushPromises()++ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({+ status: FileTaskStatus.SUCCESS,+ props: {+ directories: 2,+ files: 3,+ size: 120+ }+ })+ })+it('should create a task and mark it as error when the async method fails', async () => {vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce('22222222-2222-4222-8222-222222222222')filesMethods.doWork.mockRejectedValueOnce(new Error('operation failed'))diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.tsindex 5d096321..b0458869 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts@@ -206,7 +206,7 @@ export class FilesTasksManager implements OnModuleDestroy {taskPromise.then(async (data: any) => {this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })- const finalProps = await this.getFinalTransferProps(task, data)+ const finalProps = await this.getFinalTaskProps(task, data)this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data, finalProps).catch((e: Error) =>this.logger.warn({ tag: this.runTask.name, msg: `${e}` }))@@ -231,7 +231,10 @@ export class FilesTasksManager implements OnModuleDestroy {}}- private async getFinalTransferProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {+ private async getFinalTaskProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {+ if (task.task.type === FILE_OPERATION.DOWNLOAD || task.task.type === FILE_OPERATION.COMPRESS || task.task.type === FILE_OPERATION.DECOMPRESS) {+ return task.space.task?.props+ }if (task.task.type === FILE_OPERATION.DELETE) {const props = task.space.task?.propsreturn {@@ -248,7 +251,7 @@ export class FilesTasksManager implements OnModuleDestroy {const props = await this.filesTasksWatcher.getPathProps(publishedSpace.realPath)return { ...props, progress: 100 }} catch (e) {- this.logger.warn({ tag: this.getFinalTransferProps.name, msg: `${e}` })+ this.logger.warn({ tag: this.getFinalTaskProps.name, msg: `${e}` })}}diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.tsindex 30884f9d..e7e06f62 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts@@ -53,9 +53,28 @@ describe(FilesTasksTransfer.name, () => {await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)})+ it('tracks extracted entries and implicit parent directories', () => {+ const space = {+ task: { cacheKey, props: { progress: 1 } }+ } as any+ const onEntry = service.createExtractionProgressHandler(space)++ onEntry({ path: 'docs/guides/readme.txt', isDirectory: false, size: 7 })+ onEntry({ path: 'docs/images/', isDirectory: true, size: 0 })+ onEntry({ path: 'docs/guides/second.txt', isDirectory: false, size: 5 })+ onEntry({ path: 'docs/guides/readme.txt', isDirectory: false, size: 9 })++ expect(space.task.props).toEqual({+ progress: 1,+ files: 2,+ directories: 3,+ size: 14+ })+ })+it('publishes a copied file only after the commit hook', async () => {const signal = new AbortController().signal- let temporaryPath = ''+ const temporaryPath = path.join(stagingDir, `.${cacheKey}-destination.txt`)let transferredBytes = 0await writeFile(srcPath, 'content')@@ -65,9 +84,7 @@ describe(FilesTasksTransfer.name, () => {await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')},cacheKey,- onTransferStart: (value: string) => {- temporaryPath = value- },+ onTransferStart: vi.fn(),onProgress: (bytes: number) => {transferredBytes += bytes},@@ -95,7 +112,7 @@ describe(FilesTasksTransfer.name, () => {it('copies directly to a destination on another device', async () => {const signal = new AbortController().signal- let watchedPath = ''+ let transferStarted = falselet destinationPrepared = falseawait writeFile(srcPath, 'content')mockCrossDevice()@@ -105,15 +122,15 @@ describe(FilesTasksTransfer.name, () => {destinationPrepared = true},cacheKey,- onTransferStart: (value: string) => {+ onTransferStart: () => {expect(destinationPrepared).toBe(true)- watchedPath = value+ transferStarted = true},signal,stagingDir})- expect(watchedPath).toBe(dstPath)+ expect(transferStarted).toBe(true)await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')await expect(readdir(stagingDir)).resolves.toEqual([])})diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.tsindex 092f299a..a0de7d49 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts@@ -2,16 +2,14 @@ import { Injectable } from '@nestjs/common'import { createReadStream, createWriteStream } from 'node:fs'import fs from 'node:fs/promises'import path from 'node:path'-import { Transform } from 'node:stream'import { pipeline } from 'node:stream/promises'import { UserModel } from '../../../users/models/user.model'import { SpaceEnv } from '../../../spaces/models/space-env.model'import { DEFAULT_HIGH_WATER_MARK } from '../../constants/files'-import { FILE_OPERATION } from '../../constants/operations'import { FileTaskEvent } from '../../events/file-events'-import { fileSize, isPathExists, moveFiles, removeFiles } from '../../utils/files'-import { countDirEntriesAndSize, isCrossDeviceMove, taskTemporaryPath } from '../../utils/tasks'-import { FileTaskCopyTaskOptions, FileTaskTransferOptions } from '../../interfaces/file-task.interface'+import type { FileTaskCopyTaskOptions, FileTaskExtractionEntry, FileTaskTransferOptions } from '../../interfaces/file-task.interface'+import { createProgressTransform, fileSize, isCrossDevice, isPathExists, moveFiles, removeFiles } from '../../utils/files'+import { countDirEntriesAndSize, taskTemporaryPath } from '../../utils/tasks'import { SourceCleanupError } from '../../models/file-error'@Injectable()@@ -30,8 +28,8 @@ export class FilesTasksTransfer {await this.copyAbortable(srcSpace.realPath, dstSpace.realPath, {beforeCommit: this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination),cacheKey: srcSpace.task!.cacheKey,- onProgress: (bytes) => this.updateProgress(srcSpace, bytes),- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),+ onProgress: this.createByteProgressHandler(srcSpace),+ onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),overwrite,recursive,signal,@@ -45,24 +43,17 @@ export class FilesTasksTransfer {dstSpace: SpaceEnv,overwrite: boolean,isDir: boolean,- signal: AbortSignal | undefined,+ signal: AbortSignal,deleteDestination: () => Promise<void>): Promise<SourceCleanupError | undefined> {await this.initializeTaskProps(srcSpace, isDir)const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)- if (!signal) {- // Same-device moves stay atomic and do not copy bytes.- this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)- await beforeCommit?.()- await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)- return- }return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {beforeCommit,cacheKey: srcSpace.task!.cacheKey,crossDevice: true,- onProgress: (bytes) => this.updateProgress(srcSpace, bytes),- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),+ onProgress: this.createByteProgressHandler(srcSpace),+ onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),overwrite,signal,stagingDir: user.tasksPath@@ -80,7 +71,7 @@ export class FilesTasksTransfer {await this.initializeTaskProps(space, isDir)if (!signal) {// Moving to trash on the same device stays atomic and does not copy bytes.- this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)+ this.startTransferTaskWatch(space, trashFile)await prepareDestination()await moveFiles(space.realPath, trashFile, true)return@@ -89,8 +80,8 @@ export class FilesTasksTransfer {beforeCommit: prepareDestination,cacheKey: space.task!.cacheKey,crossDevice: true,- onProgress: (bytes) => this.updateProgress(space, bytes),- onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),+ onProgress: this.createByteProgressHandler(space),+ onTransferStart: () => this.startTransferTaskWatch(space, trashFile),overwrite: true,signal,stagingDir: user.tasksPath@@ -108,11 +99,35 @@ export class FilesTasksTransfer {}}- private updateProgress(space: SpaceEnv, bytes: number): void {- const props = space.task!.props- props.size = Math.min((props.size || 0) + bytes, props.totalSize || Number.MAX_SAFE_INTEGER)- if (props.totalSize) {- props.progress = Math.min((100 * props.size) / props.totalSize, 100)+ createByteProgressHandler(space: SpaceEnv): (bytes: number) => void {+ return (bytes) => {+ const props = space.task!.props+ props.size = Math.min((props.size ?? 0) + bytes, props.totalSize ?? Number.MAX_SAFE_INTEGER)+ if (props.totalSize) {+ props.progress = Math.min((100 * props.size) / props.totalSize, 100)+ }+ }+ }++ createExtractionProgressHandler(space: SpaceEnv): (entry: FileTaskExtractionEntry) => void {+ const directories = new Set<string>()+ const files = new Map<string, number>()+ space.task!.props = { ...space.task!.props, files: 0, directories: 0, size: 0 }+ return (entry) => {+ const parts = entry.path.split(/[/\\]/).filter((part) => part && part !== '.')+ const directoryParts = entry.isDirectory ? parts : parts.slice(0, -1)+ for (let index = 1; index <= directoryParts.length; index++) {+ directories.add(directoryParts.slice(0, index).join('/'))+ }+ const props = space.task!.props+ props.directories = directories.size+ if (!entry.isDirectory && parts.length) {+ const filePath = parts.join('/')+ const previousSize = files.get(filePath) ?? 0+ files.set(filePath, entry.size)+ props.files = files.size+ props.size = (props.size ?? 0) - previousSize + entry.size+ }}}@@ -130,8 +145,8 @@ export class FilesTasksTransfer {}}- private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {- FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)+ private startTransferTaskWatch(space: SpaceEnv, publishedPath: string): void {+ FileTaskEvent.emit('startWatch', space, publishedPath)}private async copyAbortable(srcPath: string, dstPath: string, options: FileTaskCopyTaskOptions): Promise<void> {@@ -148,7 +163,7 @@ export class FilesTasksTransfer {} = optionsconst temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)await fs.mkdir(stagingDir, { recursive: true })- const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)+ const copyDirectlyToDestination = await isCrossDevice(stagingDir, dstPath)if (copyDirectlyToDestination) {let transferStarted = false@@ -157,7 +172,7 @@ export class FilesTasksTransfer {await beforeCommit?.()signal.throwIfAborted()await this.prepareDestination(dstPath, overwrite)- onTransferStart?.(dstPath)+ onTransferStart?.()transferStarted = trueawait this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)} catch (e) {@@ -170,7 +185,7 @@ export class FilesTasksTransfer {}try {- onTransferStart?.(temporaryPath)+ onTransferStart?.()await this.copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal, onProgress)signal.throwIfAborted()await beforeCommit?.()@@ -184,7 +199,7 @@ export class FilesTasksTransfer {private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {const { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite = false, signal, stagingDir } = options- const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))+ const crossDevice = options.crossDevice ?? (await isCrossDevice(srcPath, dstPath))if (!crossDevice) {signal.throwIfAborted()await beforeCommit?.()@@ -222,13 +237,7 @@ export class FilesTasksTransfer {const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })if (onProgress) {- const progress = new Transform({- transform(chunk: Buffer, _encoding, callback) {- onProgress(chunk.length)- callback(null, chunk)- }- })- await pipeline(src, progress, dst, { signal })+ await pipeline(src, createProgressTransform(onProgress), dst, { signal })} else {await pipeline(src, dst, { signal })}diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.tsindex 9b00d02b..28202507 100644--- a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts@@ -1,6 +1,5 @@import { Cache } from '../../../../infrastructure/cache/cache.service'import { CACHE_TASK_TTL } from '../../constants/cache'-import { FILE_OPERATION } from '../../constants/operations'import { FileTaskEvent } from '../../events/file-events'import { FileTaskStatus } from '../../models/file-task'import * as filesUtils from '../../utils/files'@@ -65,22 +64,21 @@ describe(FilesTasksWatcher.name, () => {it('starts a download watcher with the published task metadata', async () => {const space = makeSpace()const updateTask = vi.spyOn(service as any, 'updateTask').mockResolvedValue(undefined)- const updateDownloadTask = vi.spyOn(service as any, 'updateDownloadTask').mockResolvedValue(undefined)const watchTask = vi.spyOn(service as any, 'watchTask').mockImplementation((_cacheKey: string, update: () => Promise<void>) => void update())- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, '/files/report.txt', '/tmp/report.part')+ FileTaskEvent.emit('startWatch', space, '/files/report.txt')await flushPromises()expect(updateTask).toHaveBeenCalledWith('task-1', {}, { name: 'report.txt', path: 'files/personal' })expect(watchTask).toHaveBeenCalledWith('task-1', expect.any(Function))- expect(updateDownloadTask).toHaveBeenCalledWith(space, '/tmp/report.part', '/files/report.txt')+ expect(updateTask).toHaveBeenCalledWith('task-1', space.task.props)})it('does not start duplicate watchers for the same task', () => {const space = makeSpace()const watchTask = vi.spyOn(service as any, 'watchTask');(service as any).tasksWatcher['task-1'] = setInterval(() => undefined, 1000)- ;(service as any).startWatch(space, FILE_OPERATION.COPY, '/files/report.txt')+ ;(service as any).startWatch(space, '/files/report.txt')expect(watchTask).not.toHaveBeenCalled()})@@ -161,21 +159,4 @@ describe(FilesTasksWatcher.name, () => {service.endFinalization('task-1')expect((service as any).finalizingTasks.has('task-1')).toBe(false)})-- it('falls back to the published file and updates download progress', async () => {- const space = makeSpace('task-1', { size: 0, totalSize: 200 })- cacheStore.set('task-1', {- props: { size: 0, totalSize: 200 },- status: FileTaskStatus.PENDING- })- vi.spyOn(filesUtils, 'isPathExists').mockImplementation(async (rPath: string) => rPath === '/files/report.txt')- vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false)- vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(50)-- await (service as any).updateDownloadTask(space, '/tmp/report.part', '/files/report.txt')-- expect(filesUtils.fileSize).toHaveBeenCalledWith('/files/report.txt')- expect(space.task.props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })- expect(cacheStore.get('task-1').props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })- })})diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.tsindex 16d09d1d..77c82db4 100644--- a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts@@ -2,10 +2,9 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'import { Cache } from '../../../../infrastructure/cache/cache.service'import { SpaceEnv } from '../../../spaces/models/space-env.model'import { CACHE_TASK_TTL } from '../../constants/cache'-import { FILE_OPERATION } from '../../constants/operations'import { FileTaskEvent } from '../../events/file-events'import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'-import { dirName, fileName, fileSize, isPathExists, isPathIsDir } from '../../utils/files'+import { dirName, fileName, fileSize, isPathIsDir } from '../../utils/files'import { countDirEntriesAndSize } from '../../utils/tasks'@Injectable()@@ -47,47 +46,23 @@ export class FilesTasksWatcher implements OnModuleDestroy {return (await isPathIsDir(rPath)) ? countDirEntriesAndSize(rPath) : { size: await fileSize(rPath) }}- private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)+ private readonly onStartWatch = (space: SpaceEnv, rPath: string): void => {+ this.startWatch(space, rPath)}- private startWatch(- space: SpaceEnv,- taskType: FILE_OPERATION,- rPath: string,- taskPath?: string,- taskName = fileName(rPath),- publishedPath?: string- ): void {+ private startWatch(space: SpaceEnv, rPath: string): void {const taskContext = space.taskif (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) returnconst { cacheKey, props } = taskContextthis.updateTask(cacheKey, props, {- name: taskName,- path: taskPath+ name: fileName(rPath),+ path: dirName(space.url)}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))- switch (taskType) {- case FILE_OPERATION.COMPRESS:- this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))- return- case FILE_OPERATION.DECOMPRESS:- this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))- return- case FILE_OPERATION.DOWNLOAD:- this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))- return- case FILE_OPERATION.COPY:- case FILE_OPERATION.DELETE:- case FILE_OPERATION.MOVE:- this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))- return- default:- this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })- }+ this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))}private watchTask(cacheKey: string, update: () => Promise<void>): void {- // Directory scans may exceed the interval, so ticks use exhaust semantics.+ // Cache updates may exceed the interval, so ticks use exhaust semantics.let updateInProgress = falseconst watcher = setInterval(() => {if (updateInProgress) return@@ -131,71 +106,4 @@ export class FilesTasksWatcher implements OnModuleDestroy {private isActiveStatus(status: FileTaskStatus): boolean {return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED}-- private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)- if (size === undefined) return- space.task.props.size = size- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)- if (props === undefined) return- space.task.props = props- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {- const isDir = await this.readWatchedPath(rPath, publishedPath, isPathIsDir)- if (isDir === undefined) return false- if (isDir) {- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)- if (props === undefined) return false- space.task.props = { ...space.task.props, ...props }- } else {- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)- if (size === undefined) return false- space.task.props.size = size- }- if (space.task.props.totalSize) {- space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize- }- return true- }-- private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {- if (!publishedPath) return read(rPath)- if (await isPathExists(rPath)) {- try {- const value = await read(rPath)- // The temporary path may be moved atomically while it is being read.- if (await isPathExists(rPath)) return value- } catch (e) {- // Retry the published path only when the temporary path disappeared during publication.- if (e?.code !== 'ENOENT') throw e- }- }- if (await isPathExists(publishedPath)) return read(publishedPath)- }}diff --git a/backend/src/applications/files/utils/download-file.spec.ts b/backend/src/applications/files/utils/download-file.spec.tsindex 5da5d430..a92ffb7f 100644--- a/backend/src/applications/files/utils/download-file.spec.ts+++ b/backend/src/applications/files/utils/download-file.spec.ts@@ -247,7 +247,7 @@ describe(DownloadFile.name, () => {maxSize: 1024})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024)+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)})it('rejects missing content-length for space downloads even when maxSize is provided', async () => {@@ -278,7 +278,21 @@ describe(DownloadFile.name, () => {})expect(space.willExceedQuota).toHaveBeenCalledWith(12)- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12)+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)+ })++ it('forwards the progress callback to the stream writer', async () => {+ const stream = Readable.from(['abc'])+ const onProgress = vi.fn()+ http.axiosRef+ .mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' }))+ .mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })++ await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {+ onProgress+ })++ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, onProgress)})it('allows zero content-length and guards the written stream at zero bytes', async () => {@@ -289,7 +303,7 @@ describe(DownloadFile.name, () => {await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0)+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0, undefined, undefined)})it('rejects redirects on GET after the HEAD URL has been resolved', async () => {@@ -318,7 +332,7 @@ describe(DownloadFile.name, () => {allowPrivateIP: true})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12)+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)expect(http.axiosRef).toHaveBeenNthCalledWith(2,expect.objectContaining({@@ -343,6 +357,6 @@ describe(DownloadFile.name, () => {maxSize: 1024})- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024)+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)})})diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.tsindex 5a2e53c2..6e05e8da 100644--- a/backend/src/applications/files/utils/download-file.ts+++ b/backend/src/applications/files/utils/download-file.ts@@ -9,7 +9,6 @@ import { AxiosHeaders, type AxiosRequestConfig, type AxiosResponse } from 'axiosimport ipaddr from 'ipaddr.js'import { HTTP_METHOD } from '../../applications.constants'import type { SpaceEnv } from '../../spaces/models/space-env.model'-import { FILE_OPERATION } from '../constants/operations'import type { DownloadFileDto } from '../dto/file-operations.dto'import { FileTaskEvent } from '../events/file-events'import type { DownloadFileContentInfo, DownloadFileOptions } from '../interfaces/download-file.interface'@@ -63,7 +62,7 @@ export class DownloadFile {if (maxSize === null || (contentLength === null && options?.space)) {throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR_MESSAGES.DOWNLOAD_INVALID_CONTENT_LENGTH)}- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath, dstPath)+ if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath)// The HEAD request resolved redirects; the GET must target that final URL directly.const { response: getRes } = await this.request(@@ -71,11 +70,7 @@ export class DownloadFile {{ method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },{ allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 })- if (options?.signal) {- await writeFromStream(dstPath, getRes.data, 0, maxSize, options.signal)- } else {- await writeFromStream(dstPath, getRes.data, 0, maxSize)- }+ await writeFromStream(dstPath, getRes.data, 0, maxSize, options?.signal, options?.onProgress)}private safeLookup(hostname: string, options: Parameters<LookupFunction>[1], cb: Parameters<LookupFunction>[2]): void {@@ -173,18 +168,14 @@ export class DownloadFile {return Number.isSafeInteger(contentLength) ? contentLength : null}- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string, dstPath: string): void {+ private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string): void {if (!space) returnif (space.willExceedQuota(contentLength)) {throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)}if (space.task?.cacheKey) {- space.task.props.totalSize = contentLength- if (publishedPath === dstPath) {- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)- } else {- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, publishedPath, dstPath)- }+ space.task.props = { ...space.task.props, progress: 1, size: 0, totalSize: contentLength }+ FileTaskEvent.emit('startWatch', space, publishedPath)}}}diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.tsindex 7d78dfff..ec668777 100644--- a/backend/src/applications/files/utils/files.spec.ts+++ b/backend/src/applications/files/utils/files.spec.ts@@ -1,11 +1,12 @@import { HttpStatus } from '@nestjs/common'-import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'+import fs, { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { Readable } from 'node:stream'+import fse from 'fs-extra'import { FileError } from '../models/file-error'import { FILE_ERROR_MESSAGES } from './errors'-import { isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'+import { isCrossDevice, isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'describe(isPathInside.name, () => {const basePath = path.join(path.sep, 'tmp', 'output')@@ -28,6 +29,23 @@ describe(isPathInside.name, () => {)})+describe(isCrossDevice.name, () => {+ afterEach(() => {+ vi.restoreAllMocks()+ })++ it('compares the source device with the nearest existing destination parent', async () => {+ vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)+ const statSpy = vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)+ vi.spyOn(fse, 'pathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(false).mockResolvedValueOnce(true)+ const dstPath = path.join(path.sep, 'missing', 'parent', 'destination.txt')++ await expect(isCrossDevice('/source.txt', dstPath)).resolves.toBe(true)++ expect(statSpy).toHaveBeenCalledWith(path.parse(dstPath).root)+ })+})+describe(writeFromStream.name, () => {let tmpDir: string@@ -75,6 +93,17 @@ describe(writeFromStream.name, () => {name: 'AbortError'})})++ it('reports bytes written through the progress callback', async () => {+ const filePath = path.join(tmpDir, 'progress.txt')+ const onProgress = vi.fn()++ await writeFromStream(filePath, Readable.from([Buffer.from('abc'), Buffer.from('de')]), 0, 5, undefined, onProgress)++ expect(onProgress).toHaveBeenCalledTimes(2)+ expect(onProgress).toHaveBeenNthCalledWith(1, 3)+ expect(onProgress).toHaveBeenNthCalledWith(2, 2)+ })})describe(makeTempDir.name, () => {diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex a3841ec0..10ce54a1 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -52,6 +52,21 @@ export function isPathExists(rPath: string): Promise<boolean> {return fse.pathExists(rPath)}+async function existingParentPath(rPath: string): Promise<string> {+ let parentPath = path.dirname(rPath)+ while (!(await isPathExists(parentPath))) {+ const nextParentPath = path.dirname(parentPath)+ if (nextParentPath === parentPath) break+ parentPath = nextParentPath+ }+ return parentPath+}++export async function isCrossDevice(srcPath: string, dstPath: string): Promise<boolean> {+ const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])+ return srcStats.dev !== dstParentStats.dev+}+export async function isPathIsReadable(rPath: string): Promise<boolean> {try {await fs.access(rPath, fs.constants.R_OK)@@ -199,23 +214,40 @@ export async function checksumFile(filePath: string, alg: string): Promise<strinreturn hash.digest('hex')}-export function writeFromStream(rPath: string, stream: Readable, start: number = 0, maxSize?: number, signal?: AbortSignal): Promise<void> {+export function createProgressTransform(onProgress: (bytes: number) => void): Transform {+ return new Transform({+ transform(chunk: Buffer, _encoding, callback) {+ onProgress(chunk.length)+ callback(null, chunk)+ }+ })+}++export function writeFromStream(+ rPath: string,+ stream: Readable,+ start: number = 0,+ maxSize?: number,+ signal?: AbortSignal,+ onProgress?: (bytes: number) => void+): Promise<void> {const dst: WriteStream = createWriteStream(rPath, { flags: start ? 'a' : 'w', start: start, highWaterMark: DEFAULT_HIGH_WATER_MARK })- if (maxSize === undefined) {+ if (maxSize === undefined && !onProgress) {return pipeline(stream, dst, { signal })}let received = start- const limitSize = new Transform({+ const progress = new Transform({transform(chunk, _encoding, callback) {received += chunk.length- if (received > maxSize) {+ if (maxSize !== undefined && received > maxSize) {callback(maxFileSizeExceededError())return}+ onProgress?.(chunk.length)callback(null, chunk)}})- return pipeline(stream, limitSize, dst, { signal })+ return pipeline(stream, progress, dst, { signal })}export async function writeFromStreamAndChecksum(rPath: string, stream: Readable, hasRange: number, alg: string): Promise<string> {diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.tsindex e159c448..d1cc10a4 100644--- a/backend/src/applications/files/utils/tasks.ts+++ b/backend/src/applications/files/utils/tasks.ts@@ -2,7 +2,7 @@ import { Dirent } from 'node:fs'import fs from 'node:fs/promises'import path from 'node:path'import { FILE_OPERATION } from '../constants/operations'-import { isPathExists, walkDir } from './files'+import { isCrossDevice, walkDir } from './files'import type { FileTaskProps } from '../models/file-task'export function taskTemporaryPrefix(cacheKey: string): string {@@ -20,21 +20,6 @@ export async function createTaskTemporaryDir(parentPath: string, cacheKey: strinreturn temporaryPath}-async function existingParentPath(dstPath: string): Promise<string> {- let parentPath = path.dirname(dstPath)- while (!(await isPathExists(parentPath))) {- const nextParentPath = path.dirname(parentPath)- if (nextParentPath === parentPath) break- parentPath = nextParentPath- }- return parentPath-}--export async function isCrossDeviceMove(srcPath: string, dstPath: string): Promise<boolean> {- const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])- return srcStats.dev !== dstParentStats.dev-}-export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dstPath?: string): Promise<boolean> {switch (type) {case FILE_OPERATION.COPY:@@ -46,7 +31,7 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dcase FILE_OPERATION.DELETE:if (!dstPath) return falsetry {- return await isCrossDeviceMove(srcPath, dstPath)+ return await isCrossDevice(srcPath, dstPath)} catch {return false}diff --git a/backend/src/applications/files/utils/untar-file.spec.ts b/backend/src/applications/files/utils/untar-file.spec.tsindex 052b6d8c..e7eb1b23 100644--- a/backend/src/applications/files/utils/untar-file.spec.ts+++ b/backend/src/applications/files/utils/untar-file.spec.ts@@ -3,7 +3,7 @@ import { access, mkdir, mkdtemp, readlink, rm, symlink, writeFile } from 'node:fimport os from 'node:os'import path from 'node:path'import { create } from 'tar'-import { checkTarEntry, extractTar } from './untar-file'+import { checkTarEntry, extractTar, isTarDirectory } from './untar-file'describe(extractTar.name, () => {const baseOutputDir = path.join(path.sep, 'tmp', 'output')@@ -20,6 +20,12 @@ describe(extractTar.name, () => {)})+ it('recognizes regular and GNU dump directories', () => {+ expect(isTarDirectory('Directory')).toBe(true)+ expect(isTarDirectory('GNUDumpDir')).toBe(true)+ expect(isTarDirectory('File')).toBe(false)+ })+let tmpDir: stringlet sourceDir: stringlet outputDir: string@@ -40,12 +46,26 @@ describe(extractTar.name, () => {})it('extracts symbolic links targeting the output directory', async () => {+ const onEntry = vi.fn()await symlink('./v2', path.join(sourceDir, 'docs', 'latest'))await createArchive()- await extractTar(archivePath, outputDir, false)+ await extractTar(archivePath, outputDir, false, undefined, undefined, onEntry)await expect(readlink(path.join(outputDir, 'docs', 'latest'))).resolves.toBe('./v2')+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/', isDirectory: true, size: 0 })+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/latest', isDirectory: false, size: 0 })+ })++ it('reports decompressed file bytes through the entry transform', async () => {+ const onEntry = vi.fn()+ await writeFile(path.join(sourceDir, 'docs', 'file.txt'), 'abc')+ await createArchive()++ await extractTar(archivePath, outputDir, false, undefined, undefined, onEntry)++ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/file.txt', isDirectory: false, size: 0 })+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/file.txt', isDirectory: false, size: 3 })})it('rejects symbolic links escaping the output directory', async () => {diff --git a/backend/src/applications/files/utils/untar-file.ts b/backend/src/applications/files/utils/untar-file.tsindex e824fe24..411fe147 100644--- a/backend/src/applications/files/utils/untar-file.ts+++ b/backend/src/applications/files/utils/untar-file.ts@@ -2,8 +2,9 @@ import { createReadStream } from 'node:fs'import path from 'node:path'import { pipeline } from 'node:stream/promises'import { extract, type ReadEntry, type Unpack } from 'tar'+import type { FileTaskExtractionEntry } from '../interfaces/file-task.interface'import { storageQuotaExceededError } from './errors'-import { isPathInside } from './files'+import { createProgressTransform, isPathInside } from './files'export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' | 'path' | 'linkpath'>): boolean {if (entry.type === 'Link') {@@ -21,7 +22,18 @@ export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' |return true}-export async function extractTar(filePath: string, outputDir: string, gzip: boolean, maxExtractedSize?: number, signal?: AbortSignal): Promise<void> {+export function isTarDirectory(type: ReadEntry['type']): boolean {+ return type === 'Directory' || type === 'GNUDumpDir'+}++export async function extractTar(+ filePath: string,+ outputDir: string,+ gzip: boolean,+ maxExtractedSize?: number,+ signal?: AbortSignal,+ onEntry?: (entry: FileTaskExtractionEntry) => void+): Promise<void> {let validationError: Error | undefinedlet extractedSize = 0const srcStream = createReadStream(filePath)@@ -40,6 +52,15 @@ export async function extractTar(filePath: string, outputDir: string, gzip: boolpreserveOwner: false,preservePaths: false,strict: true,+ transform: onEntry+ ? (entry) => {+ let extractedEntrySize = 0+ return createProgressTransform((bytes) => {+ extractedEntrySize += bytes+ onEntry({ path: entry.path, isDirectory: false, size: extractedEntrySize })+ }) as unknown as ReadEntry+ }+ : undefined,filter: (_path, entry) => {if (validationError) return falsetry {@@ -48,7 +69,10 @@ export async function extractTar(filePath: string, outputDir: string, gzip: boolif (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {throw storageQuotaExceededError()}- return !('type' in entry) || checkTarEntry(outputDir, entry)+ if (!('type' in entry)) return true+ if (!checkTarEntry(outputDir, entry)) return false+ onEntry?.({ path: entry.path, isDirectory: isTarDirectory(entry.type), size: 0 })+ return true} catch (e) {return abortExtraction(e as Error)}diff --git a/backend/src/applications/files/utils/unzip-file.spec.ts b/backend/src/applications/files/utils/unzip-file.spec.tsindex 3c401e55..2b0eeb90 100644--- a/backend/src/applications/files/utils/unzip-file.spec.ts+++ b/backend/src/applications/files/utils/unzip-file.spec.ts@@ -58,10 +58,13 @@ describe(extractZip.name, () => {it('extracts entries inside the output directory', async () => {const extractedPath = path.join(outputDir, 'safe.txt')- await writeFile(archivePath, createZip('safe.txt'))+ const onEntry = vi.fn()+ await writeFile(archivePath, createZip('safe.txt', Buffer.from('abc')))- await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()+ await expect(extractZip(archivePath, outputDir, undefined, undefined, onEntry)).resolves.toBeUndefined()await expect(access(extractedPath)).resolves.toBeUndefined()+ expect(onEntry).toHaveBeenCalledWith({ path: 'safe.txt', isDirectory: false, size: 0 })+ expect(onEntry).toHaveBeenCalledWith({ path: 'safe.txt', isDirectory: false, size: 3 })})it('rejects entries outside the output directory', async () => {diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.tsindex 574c11a7..9ac41ffa 100644--- a/backend/src/applications/files/utils/unzip-file.ts+++ b/backend/src/applications/files/utils/unzip-file.ts@@ -4,12 +4,19 @@ import { pipeline } from 'node:stream/promises'import { promisify } from 'node:util'import { Entry, open as openZip, Options, ZipFile } from 'yauzl'import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'+import type { FileTaskExtractionEntry } from '../interfaces/file-task.interface'import { storageQuotaExceededError } from './errors'-import { isPathInside, makeDir } from './files'+import { createProgressTransform, isPathInside, makeDir } from './files'const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)-export async function extractZip(filePath: string, outputDir: string, maxExtractedSize?: number, signal?: AbortSignal): Promise<void> {+export async function extractZip(+ filePath: string,+ outputDir: string,+ maxExtractedSize?: number,+ signal?: AbortSignal,+ onEntry?: (entry: FileTaskExtractionEntry) => void+): Promise<void> {// Reject entries whose actual decompressed size differs from their metadata.const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))@@ -44,12 +51,28 @@ export async function extractZip(filePath: string, outputDir: string, maxExtract}if (isDir) {await makeDir(fullPath, true)+ onEntry?.({ path: entry.fileName, isDirectory: true, size: 0 })zipFile.readEntry()} else {// make sure parent existsawait makeDir(path.dirname(fullPath), true)const readStream = await openReadStream(entry)- await pipeline(readStream, fs.createWriteStream(fullPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK }), { signal })+ const writeStream = fs.createWriteStream(fullPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })+ if (onEntry) {+ let extractedEntrySize = 0+ onEntry({ path: entry.fileName, isDirectory: false, size: 0 })+ await pipeline(+ readStream,+ createProgressTransform((bytes) => {+ extractedEntrySize += bytes+ onEntry({ path: entry.fileName, isDirectory: false, size: extractedEntrySize })+ }),+ writeStream,+ { signal }+ )+ } else {+ await pipeline(readStream, writeStream, { signal })+ }zipFile.readEntry()}} catch (err) {