Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): improve task progress tracking for copy and move operations
johaven79394910fd94
diff --git a/backend/src/applications/files/files-tasks.controller.spec.ts b/backend/src/applications/files/files-tasks.controller.spec.tsindex 0b49579d..2c95f7fc 100644--- a/backend/src/applications/files/files-tasks.controller.spec.ts+++ b/backend/src/applications/files/files-tasks.controller.spec.ts@@ -2,7 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing'import { Cache } from '../../infrastructure/cache/cache.service'import { FilesTasksController } from './files-tasks.controller'import { FilesMethods } from './services/files-methods.service'-import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'import { Mock } from 'vitest'describe(FilesTasksController.name, () => {diff --git a/backend/src/applications/files/files-tasks.controller.ts b/backend/src/applications/files/files-tasks.controller.tsindex 862ac07f..fb1c311b 100644--- a/backend/src/applications/files/files-tasks.controller.ts+++ b/backend/src/applications/files/files-tasks.controller.ts@@ -6,7 +6,7 @@ import { UserModel } from '../users/models/user.model'import { API_FILES_TASKS, FILES_ROUTE } from './constants/routes'import { FileTasksPollDto } from './dto/file-tasks-poll.dto'import type { FileTasksPollResponse } from './interfaces/file-task.interface'-import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'@Controller(API_FILES_TASKS)export class FilesTasksController {diff --git a/backend/src/applications/files/files.controller.spec.ts b/backend/src/applications/files/files.controller.spec.tsindex 9d2dd7ee..665d5a95 100644--- a/backend/src/applications/files/files.controller.spec.ts+++ b/backend/src/applications/files/files.controller.spec.ts@@ -7,7 +7,7 @@ import { FilesController } from './files.controller'import { FilesMethods } from './services/files-methods.service'import { FilesRecents } from './services/files-recents.service'import { FilesSearchManager } from './services/files-search-manager.service'-import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'import { FilesContentIndexer } from './services/files-content-indexer.service'describe(FilesController.name, () => {diff --git a/backend/src/applications/files/files.controller.ts b/backend/src/applications/files/files.controller.tsindex 72ec125a..8764d7fd 100644--- a/backend/src/applications/files/files.controller.ts+++ b/backend/src/applications/files/files.controller.ts@@ -43,7 +43,7 @@ import { FileRecent } from './schemas/file-recent.interface'import { FilesMethods } from './services/files-methods.service'import { FilesRecents } from './services/files-recents.service'import { FilesSearchManager } from './services/files-search-manager.service'-import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'import { FilesContentIndexer } from './services/files-content-indexer.service'import { UserHaveRole } from '../users/decorators/roles.decorator'import { USER_ROLE } from '../users/constants/user'diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex 919c2628..5397c78e 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -15,8 +15,10 @@ import { FilesQueries } from './services/files-queries.service'import { FilesRecents } from './services/files-recents.service'import { FilesScheduler } from './services/files-scheduler.service'import { FilesSearchManager } from './services/files-search-manager.service'-import { FilesTasksManager } from './services/files-tasks-manager.service'-import { FilesTasksQueue } from './services/files-tasks-queue.service'+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'+import { FilesTasksQueue } from './services/tasks/files-tasks-queue.service'+import { FilesTasksTransfer } from './services/tasks/files-tasks-transfer.service'+import { FilesTasksWatcher } from './services/tasks/files-tasks-watcher.service'import { FilesEventManager } from './services/files-event-manager.service'import { FilesQuotaManager } from './services/files-quota-manager.service'import { FilesTrashRetention } from './services/files-trash-retention.service'@@ -34,6 +36,8 @@ import { FilesTrashRetention } from './services/files-trash-retention.service'FilesLockManager,FilesTasksManager,FilesTasksQueue,+ FilesTasksTransfer,+ FilesTasksWatcher,FilesScheduler,FilesRecents,FilesContentParser,diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.tsindex 6cdb7c1e..82b5efc9 100644--- a/backend/src/applications/files/interfaces/file-task.interface.ts+++ b/backend/src/applications/files/interfaces/file-task.interface.ts@@ -5,3 +5,18 @@ export interface FileTasksPollResponse {ended: FileTask[]missingIds: string[]}+export interface FileTaskTransferOptions {+ beforeCommit?: () => Promise<void>+ cacheKey: string+ crossDevice?: boolean+ onTransferStart?: (temporaryPath: string) => void+ onProgress?: (bytes: number) => void+ overwrite?: boolean+ signal: AbortSignal+ stagingDir?: string+}++export interface FileTaskCopyTaskOptions extends FileTaskTransferOptions {+ preserveTimestamps?: boolean+ recursive?: boolean+}diff --git a/backend/src/applications/files/models/file-error.ts b/backend/src/applications/files/models/file-error.tsindex 0df437d1..0cfcf61f 100644--- a/backend/src/applications/files/models/file-error.ts+++ b/backend/src/applications/files/models/file-error.ts@@ -7,3 +7,15 @@ export class FileError extends Error {this.httpCode = httpCode}}++export class SourceCleanupError extends Error {+ // The destination is committed, but the obsolete source still requires manual or deferred cleanup.+ constructor(+ readonly srcPath: string,+ readonly dstPath: string,+ options: ErrorOptions+ ) {+ super('Destination was published but the source could not be removed', options)+ this.name = SourceCleanupError.name+ }+}diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex abee5c11..a80f6b00 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -18,7 +18,7 @@ import { ACTION } from '../../../common/constants'import { FILE_OPERATION } from '../constants/operations'import { DownloadFileDto } from '../dto/file-operations.dto'import { FileEvent, FileTaskEvent } from '../events/file-events'-import { FileError } from '../models/file-error'+import { FileError, SourceCleanupError } from '../models/file-error'import { LockConflict } from '../models/file-lock-error'import { FILE_ERROR_MESSAGES } from '../utils/errors'import { SendFile } from '../utils/send-file'@@ -29,6 +29,7 @@ import * as taskUtils from '../utils/tasks'import { FilesLockManager } from './files-lock-manager.service'import { FilesManager } from './files-manager.service'import { FilesQueries } from './files-queries.service'+import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'import { Mock } from 'vitest'vi.mock('archiver', () => ({@@ -42,6 +43,7 @@ vi.mock('node:dns/promises', () => ({describe(FilesManager.name, () => {let service: FilesManager+ let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock }let http: { axiosRef: Mock }const lookupMock = lookup as Mocklet filesQueries: { moveFiles: Mock; deleteFiles: Mock }@@ -127,8 +129,9 @@ describe(FilesManager.name, () => {expect(filesUtils.copyFiles).not.toHaveBeenCalled()expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesUtils.removeFiles).not.toHaveBeenCalled()- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()+ expect(filesTasksTransfer.delete).not.toHaveBeenCalled()expect(filesLockManager.create).not.toHaveBeenCalled()expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()expect(filesLockManager.checkConflicts).not.toHaveBeenCalled()@@ -150,6 +153,27 @@ describe(FilesManager.name, () => {notificationsManager = {create: vi.fn().mockResolvedValue(undefined)}+ filesTasksTransfer = {+ copy: vi+ .fn()+ .mockImplementation(+ async (+ _user: any,+ srcSpace: any,+ _dstSpace: any,+ overwrite: boolean,+ _recursive: boolean,+ _isDir: boolean,+ _signal: AbortSignal,+ deleteDestination: () => Promise<void>+ ) => {+ srcSpace.task.props = { ...srcSpace.task.props, progress: 40, size: 40, totalSize: 100 }+ if (overwrite) await deleteDestination()+ }+ ),+ move: vi.fn().mockResolvedValue(undefined),+ delete: vi.fn().mockResolvedValue(undefined)+ }filesLockManager = {create: vi.fn().mockResolvedValue([true, { key: 'lock-1' }]),checkConflicts: vi.fn().mockResolvedValue(undefined),@@ -168,6 +192,7 @@ describe(FilesManager.name, () => {{ provide: NotificationsManager, useValue: notificationsManager },{ provide: HttpService, useValue: http },{ provide: FilesLockManager, useValue: filesLockManager },+ { provide: FilesTasksTransfer, useValue: filesTasksTransfer },FilesManager]}).compile()@@ -192,16 +217,7 @@ describe(FilesManager.name, () => {vi.spyOn(filesUtils, 'dirSize').mockResolvedValue([123, {}] as any)vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/unique-path.txt')vi.spyOn(filesUtils, 'uniqueDatedFilePath').mockResolvedValue({ isDir: false, path: '/trash/file-2026.txt' })- vi.spyOn(taskUtils, 'copyAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {- options.onTransferStart?.('/tmp/task-transfer')- await options.beforeCommit?.()- })vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue('/data/users/john/tmp/tasks/.task-d-archive')- vi.spyOn(taskUtils, 'moveAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {- options.onTransferStart?.('/tmp/task-transfer')- await options.beforeCommit?.()- return undefined- })vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>path.join(parentPath, `.${cacheKey}-${path.basename(name)}`))@@ -812,7 +828,7 @@ describe(FilesManager.name, () => {expect(filesLockManager.checkConflicts).toHaveBeenCalledWith(dst.dbFile, DEPTH.RESOURCE, { userId: 7, lockTokens: undefined })expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false, false)- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: dst.realPath })})@@ -846,7 +862,7 @@ describe(FilesManager.name, () => {await service.copyMove(user, src, dst, true)expect(filesUtils.moveFiles).toHaveBeenCalledWith('/src-base/src.txt', '/dst-base/dst.txt', false)- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)expect(emitSpy).toHaveBeenCalledWith('event', { user, space: src, action: ACTION.DELETE_PERMANENTLY, rPath: '/src-base/src.txt' })expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: '/dst-base/dst.txt' })@@ -865,22 +881,13 @@ describe(FilesManager.name, () => {dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }})const signal = new AbortController().signal- const cleanupError = new taskUtils.SourceCleanupError(src.realPath, dst.realPath, { cause: new Error('cleanup failed') })+ const cleanupError = new SourceCleanupError(src.realPath, dst.realPath, { cause: new Error('cleanup failed') })prepareFileTransfer(src.realPath, dst.realPath)- vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)+ filesTasksTransfer.move.mockResolvedValueOnce(cleanupError)await expect(service.copyMove(user, src, dst, true, false, false, undefined, signal)).rejects.toBe(cleanupError)- expect(taskUtils.moveAbortable).toHaveBeenCalledWith(- src.realPath,- dst.realPath,- expect.objectContaining({- cacheKey: 'task-move',- crossDevice: true,- signal,- stagingDir: user.tasksPath- })- )+ expect(filesTasksTransfer.move).toHaveBeenCalledWith(user, src, dst, false, false, signal, expect.any(Function))expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)})@@ -900,7 +907,7 @@ describe(FilesManager.name, () => {await service.copyMove(user, src, dst, true)expect(filesUtils.moveFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false)- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()})it('should preserve the regular overwrite path outside a task context', async () => {@@ -919,7 +926,7 @@ describe(FilesManager.name, () => {expect(deleteSpy).toHaveBeenCalledWith(user, dst)expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, true, false)- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()expect(deleteSpy.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.copyFiles).mock.invocationCallOrder[0])})@@ -939,18 +946,9 @@ describe(FilesManager.name, () => {await service.copyMove(user, src, dst, false, true, false, undefined, signal)- expect(taskUtils.copyAbortable).toHaveBeenCalledWith(- src.realPath,- dst.realPath,- expect.objectContaining({- beforeCommit: expect.any(Function),- cacheKey: 'task-copy',- overwrite: true,- signal,- stagingDir: user.tasksPath- })- )+ expect(filesTasksTransfer.copy).toHaveBeenCalledWith(user, src, dst, true, false, false, signal, expect.any(Function))expect(deleteSpy).toHaveBeenCalledWith(user, dst)+ expect(src.task.props).toMatchObject({ progress: 40, size: 40, totalSize: 100 })})})@@ -991,23 +989,13 @@ describe(FilesManager.name, () => {})const trashFile = '/data/users/john/trash/documents/document.txt'const signal = new AbortController().signal- const cleanupError = new taskUtils.SourceCleanupError(space.realPath, trashFile, { cause: new Error('cleanup failed') })+ const cleanupError = new SourceCleanupError(space.realPath, trashFile, { cause: new Error('cleanup failed') })prepareFileTransfer(space.realPath, trashFile)- vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)+ filesTasksTransfer.delete.mockResolvedValueOnce(cleanupError)await expect(service.delete(user, space, undefined, signal)).rejects.toBe(cleanupError)- expect(taskUtils.moveAbortable).toHaveBeenCalledWith(- space.realPath,- trashFile,- expect.objectContaining({- cacheKey: 'task-delete',- onTransferStart: expect.any(Function),- overwrite: true,- signal,- stagingDir: user.tasksPath- })- )+ expect(filesTasksTransfer.delete).toHaveBeenCalledWith(user, space, trashFile, false, signal, expect.any(Function))expect(filesUtils.moveFiles).not.toHaveBeenCalled()expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, false)})@@ -1023,7 +1011,7 @@ describe(FilesManager.name, () => {await service.delete(user, space)expect(filesUtils.moveFiles).toHaveBeenCalledWith(space.realPath, trashFile, true)- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()+ expect(filesTasksTransfer.delete).not.toHaveBeenCalled()})})diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex be7ec99c..8997308e 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -29,7 +29,7 @@ import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'import { FileDBProps } from '../interfaces/file-db-props.interface'import { FileLock } from '../interfaces/file-lock.interface'import { FileLockProps } from '../interfaces/file-props.interface'-import { FileError } from '../models/file-error'+import { FileError, SourceCleanupError } from '../models/file-error'import { LockConflict } from '../models/file-lock-error'import {checkFileName,@@ -66,7 +66,8 @@ import { ACTION } from '../../../common/constants'import { pipeline } from 'node:stream/promises'import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'import { FILE_ERROR_MESSAGES, maxFileSizeExceededError } from '../utils/errors'-import { copyAbortable, createTaskTemporaryDir, moveAbortable, SourceCleanupError, taskTemporaryPath } from '../utils/tasks'+import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'+import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'@Injectable()export class FilesManager {@@ -79,7 +80,8 @@ export class FilesManager {private readonly spacesManager: SpacesManager,private readonly contextManager: ContextManager,private readonly notificationsManager: NotificationsManager,- public readonly filesLockManager: FilesLockManager+ public readonly filesLockManager: FilesLockManager,+ private readonly filesTasksTransfer: FilesTasksTransfer) {}sendFileFromSpace(space: SpaceEnv, downloadName = ''): SendFile {@@ -387,6 +389,7 @@ export class FilesManager {signal?: AbortSignal): Promise<void> {const isTaskContext = Boolean(srcSpace.task?.cacheKey)+ const useTaskTransfer = isTaskContext && (!isMove || Boolean(signal))// checksthis.checkNotTrashRepository(dstSpace)if (!canAccessToSpace(user, dstSpace)) {@@ -476,15 +479,15 @@ export class FilesManager {await this.filesLockManager.checkConflicts(dstSpace.dbFile, depth, { userId: user.id, lockTokens: dav?.lockTokens })// Task transfers defer overwrite handling until their staged content is ready to commit.- if (!isTaskContext && overwrite && (await isPathExists(dstSpace.realPath))) {+ if (!useTaskTransfer && overwrite && (await isPathExists(dstSpace.realPath))) {await this.delete(user, dstSpace)}// doif (isMove) {let sourceCleanupError: SourceCleanupError | undefined- if (isTaskContext) {- sourceCleanupError = await this.moveAsTask(user, srcSpace, dstSpace, overwrite, isDir, signal)+ if (useTaskTransfer) {+ sourceCleanupError = await this.filesTasksTransfer.move(user, srcSpace, dstSpace, overwrite, isDir, signal, () => this.delete(user, dstSpace))} else {await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)}@@ -504,7 +507,7 @@ export class FilesManager {if (!signal) {throw new Error('An abort signal is required for a copy task')}- await this.copyAsTask(user, srcSpace, dstSpace, overwrite, recursive, isDir, signal)+ await this.filesTasksTransfer.copy(user, srcSpace, dstSpace, overwrite, recursive, isDir, signal, () => this.delete(user, dstSpace))} else {await copyFiles(srcSpace.realPath, dstSpace.realPath, overwrite, recursive)}@@ -540,7 +543,9 @@ export class FilesManager {await makeDir(trashDir, true)}if (isTaskContext) {- sourceCleanupError = await this.deleteAsTask(user, space, trashFile, isDir, signal)+ sourceCleanupError = await this.filesTasksTransfer.delete(user, space, trashFile, isDir, signal, () =>+ this.moveExistingTrashFile(space, trashFile)+ )} else {await this.moveExistingTrashFile(space, trashFile)await moveFiles(space.realPath, trashFile, true)@@ -821,92 +826,6 @@ export class FilesManager {}}- private async copyAsTask(- user: UserModel,- srcSpace: SpaceEnv,- dstSpace: SpaceEnv,- overwrite: boolean,- recursive: boolean,- isDir: boolean,- signal: AbortSignal- ): Promise<void> {- await this.setTaskTotalSize(srcSpace, isDir)- await copyAbortable(srcSpace.realPath, dstSpace.realPath, {- beforeCommit: this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite),- cacheKey: srcSpace.task!.cacheKey,- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),- overwrite,- recursive,- signal,- stagingDir: user.tasksPath- })- }-- private async moveAsTask(- user: UserModel,- srcSpace: SpaceEnv,- dstSpace: SpaceEnv,- overwrite: boolean,- isDir: boolean,- signal?: AbortSignal- ): Promise<SourceCleanupError | undefined> {- await this.setTaskTotalSize(srcSpace, isDir)- const beforeCommit = this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite)- if (!signal) {- // Same-device moves stay on the regular atomic path and are therefore not cancellable.- this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)- await beforeCommit?.()- await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)- return- }- return moveAbortable(srcSpace.realPath, dstSpace.realPath, {- beforeCommit,- cacheKey: srcSpace.task!.cacheKey,- crossDevice: true,- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),- overwrite,- signal,- stagingDir: user.tasksPath- })- }-- private async deleteAsTask(- user: UserModel,- space: SpaceEnv,- trashFile: string,- isDir: boolean,- signal?: AbortSignal- ): Promise<SourceCleanupError | undefined> {- await this.setTaskTotalSize(space, isDir)- const beforeCommit = () => this.moveExistingTrashFile(space, trashFile)- if (!signal) {- // Moving to trash on the same device stays atomic; only cross-device deletes receive a signal.- this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)- await beforeCommit()- await moveFiles(space.realPath, trashFile, true)- return- }- return moveAbortable(space.realPath, trashFile, {- beforeCommit,- cacheKey: space.task!.cacheKey,- crossDevice: true,- onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),- overwrite: true,- signal,- stagingDir: user.tasksPath- })- }-- private prepareTaskDestination(user: UserModel, srcSpace: SpaceEnv, dstSpace: SpaceEnv, overwrite: boolean): (() => Promise<void>) | undefined {- if (!overwrite || srcSpace.realPath.toLowerCase() === dstSpace.realPath.toLowerCase()) return- // Preserve the current destination until the staged transfer reaches its commit phase.- return async () => {- if (await isPathExists(dstSpace.realPath)) {- await this.delete(user, dstSpace)- }- }- }-private async moveExistingTrashFile(space: SpaceEnv, trashFile: string): Promise<void> {if (!(await isPathExists(trashFile))) returnconst dstTrash = await uniqueDatedFilePath(trashFile)@@ -916,16 +835,6 @@ export class FilesManager {await this.filesQueries.moveFiles(trashFileDB, dstTrashFileDB, dstTrash.isDir)}- private async setTaskTotalSize(space: SpaceEnv, isDir: boolean): Promise<void> {- if (!isDir) {- space.task!.props.totalSize = await fileSize(space.realPath)- }- }-- private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {- FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)- }-private logSourceCleanupError(error: SourceCleanupError): void {this.logger.error({tag: this.logSourceCleanupError.name,diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 9924f45a..64873beb 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -19,7 +19,7 @@ import { files } from '../schemas/files.schema'import { isPathExists, removeFiles } from '../utils/files'import { taskTemporaryPrefix } from '../utils/tasks'import { FilesContentIndexer } from './files-content-indexer.service'-import { FilesTasksManager } from './files-tasks-manager.service'+import { FilesTasksManager } from './tasks/files-tasks-manager.service'import { FilesQuotaManager } from './files-quota-manager.service'import { FilesTrashRetention } from './files-trash-retention.service'diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.tssimilarity index 84%rename from backend/src/applications/files/services/files-tasks-manager.service.spec.tsrename to backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.tsindex 9b9c9037..becac1ba 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts@@ -1,21 +1,24 @@import { Test, TestingModule } from '@nestjs/testing'import { HttpException, HttpStatus } from '@nestjs/common'import crypto from 'node:crypto'-import { Cache } from '../../../infrastructure/cache/cache.service'-import { SpacesManager } from '../../spaces/services/spaces-manager.service'-import * as filesUtils from '../utils/files'-import { FileTaskEvent } from '../events/file-events'-import { SendFile } from '../utils/send-file'-import { FILE_OPERATION } from '../constants/operations'-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'-import { FileTaskStatus } from '../models/file-task'-import { FilesMethods } from './files-methods.service'+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { SpacesManager } from '../../../spaces/services/spaces-manager.service'+import * as filesUtils from '../../utils/files'+import * as tasksUtils from '../../utils/tasks'+import { FileTaskEvent } from '../../events/file-events'+import { SendFile } from '../../utils/send-file'+import { FILE_OPERATION } from '../../constants/operations'+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'+import { FileTaskStatus } from '../../models/file-task'+import { FilesMethods } from '../files-methods.service'import { FilesTasksManager } from './files-tasks-manager.service'import { FilesTasksQueue } from './files-tasks-queue.service'+import { FilesTasksWatcher } from './files-tasks-watcher.service'import { Mock } from 'vitest'describe(FilesTasksManager.name, () => {let filesTasksManager: FilesTasksManager+ let filesTasksWatcher: FilesTasksWatcherlet module: TestingModulelet filesMethods: {doWork: Mock@@ -90,6 +93,7 @@ describe(FilesTasksManager.name, () => {providers: [FilesTasksManager,FilesTasksQueue,+ FilesTasksWatcher,{provide: Cache,useValue: cache@@ -101,6 +105,7 @@ describe(FilesTasksManager.name, () => {module.useLogger(['fatal'])filesTasksManager = module.get<FilesTasksManager>(FilesTasksManager)+ filesTasksWatcher = module.get<FilesTasksWatcher>(FilesTasksWatcher)})afterEach(async () => {@@ -118,14 +123,14 @@ describe(FilesTasksManager.name, () => {const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')const space = { task: { cacheKey, props: {} } } as anycacheStore.set(cacheKey, { props: {} })- ;(filesTasksManager as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')+ ;(filesTasksWatcher as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')- const watcher = (filesTasksManager as any).tasksWatcher[cacheKey] as NodeJS.Timeout+ const watcher = (filesTasksWatcher as any).tasksWatcher[cacheKey] as NodeJS.Timeoutexpect(watcher.hasRef()).toBe(false)- filesTasksManager.onModuleDestroy()+ filesTasksWatcher.onModuleDestroy()- expect((filesTasksManager as any).tasksWatcher).toEqual({})+ expect((filesTasksWatcher as any).tasksWatcher).toEqual({})expect(FileTaskEvent.listenerCount('startWatch')).toBe(listenersBeforeDestroy - 1)})@@ -139,7 +144,7 @@ describe(FilesTasksManager.name, () => {finishUpdate = resolve}))- ;(filesTasksManager as any).watchTask('task-1', update)+ ;(filesTasksWatcher as any).watchTask('task-1', update)await vi.advanceTimersByTimeAsync(2000)expect(update).toHaveBeenCalledTimes(1)@@ -148,7 +153,7 @@ describe(FilesTasksManager.name, () => {await Promise.resolve()await vi.advanceTimersByTimeAsync(1000)expect(update).toHaveBeenCalledTimes(2)- ;(filesTasksManager as any).stopWatch('task-1')+ ;(filesTasksWatcher as any).stopWatch('task-1')} finally {vi.useRealTimers()}@@ -214,7 +219,7 @@ describe(FilesTasksManager.name, () => {})it('should watch a temporary path while keeping the published file name', () => {- const startWatchSpy = vi.spyOn(filesTasksManager as any, 'startWatch').mockResolvedValueOnce(undefined)+ const startWatchSpy = vi.spyOn(filesTasksWatcher as any, 'startWatch').mockResolvedValueOnce(undefined)const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as anyFileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')@@ -232,10 +237,10 @@ describe(FilesTasksManager.name, () => {it('should watch the published archive after the temporary archive has been moved', async () => {const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)const space = { task: { cacheKey: 'task-1', props: {} } } as any- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')@@ -248,10 +253,10 @@ describe(FilesTasksManager.name, () => {const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)const space = { task: { cacheKey: 'task-1', props: {} } } as any- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')@@ -264,32 +269,32 @@ describe(FilesTasksManager.name, () => {it('should retry the published directory when the temporary directory is moved while reading it', async () => {const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)const countDirEntriesSpy = vi- .spyOn(filesUtils, 'countDirEntries')- .mockResolvedValueOnce({ files: 0, directories: 0 })- .mockResolvedValueOnce({ files: 2, directories: 1 })- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ .spyOn(tasksUtils, 'countDirEntriesAndSize')+ .mockResolvedValueOnce({ files: 0, directories: 0, size: 0 })+ .mockResolvedValueOnce({ files: 2, directories: 1, size: 42 })+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)const space = { task: { cacheKey: 'task-1', props: {} } } as any- await (filesTasksManager as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')+ await (filesTasksWatcher as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')- expect(space.task.props).toEqual({ files: 2, directories: 1 })- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1 })+ expect(space.task.props).toEqual({ files: 2, directories: 1, size: 42 })+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1, size: 42 })})it('should silently skip watcher update while temporary and published paths are missing', async () => {vi.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize')- const loggerErrorSpy = vi.spyOn((filesTasksManager as any).logger, 'error')- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)+ const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)const space = { task: { cacheKey: 'task-1', props: {} } } as any- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')expect(loggerErrorSpy).not.toHaveBeenCalled()expect(stopWatchSpy).not.toHaveBeenCalled()@@ -300,11 +305,11 @@ describe(FilesTasksManager.name, () => {it('should stop a watcher without staging when its path is missing', async () => {const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)- const loggerErrorSpy = vi.spyOn((filesTasksManager as any).logger, 'error')- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)+ const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)const space = { task: { cacheKey: 'task-1', props: {} } } as any- await (filesTasksManager as any).updateCompressTask(space, '/files/archive.tgz')+ await (filesTasksWatcher as any).updateCompressTask(space, '/files/archive.tgz')expect(loggerErrorSpy).toHaveBeenCalled()expect(stopWatchSpy).toHaveBeenCalledWith('task-1')@@ -331,7 +336,7 @@ describe(FilesTasksManager.name, () => {return cacheStore.get(key)})- const updatePromise = (filesTasksManager as any).updateTask(cacheKey, { size: 20 })+ const updatePromise = (filesTasksWatcher as any).updateTask(cacheKey, { size: 20 })await getStartedconst completionPromise = (filesTasksManager as any).setTaskDone(cacheKey, FileTaskStatus.SUCCESS, 'done')releaseGet()@@ -367,6 +372,67 @@ describe(FilesTasksManager.name, () => {expect(storedTask.endedAt).toBeDefined()})+ it('should recalculate final copy properties from the published path', async () => {+ const taskId = '77777777-7777-4777-8777-777777777777'+ const user = { id: 7 } as any+ const space = { url: 'files/personal/source' } as any+ const result = { path: 'files/personal', name: 'destination', mime: 'directory' }+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)+ filesMethods.doWork.mockResolvedValueOnce(result)+ const getPathPropsSpy = vi.spyOn(filesTasksWatcher, 'getPathProps').mockResolvedValueOnce({ directories: 2, files: 3, size: 120 })++ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, space, null, 'doWork')+ await flushPromises()++ expect(spacesManager.spaceEnv).toHaveBeenCalledWith(user, ['files', 'personal', 'destination'])+ expect(getPathPropsSpy).toHaveBeenCalledWith('/data/users/john/files/destination.txt')+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({+ status: FileTaskStatus.SUCCESS,+ path: result.path,+ name: result.name,+ props: {+ directories: 2,+ files: 3,+ progress: 100,+ size: 120+ }+ })+ })++ it('should finalize a fast delete task without waiting for a watcher tick', async () => {+ const taskId = '66666666-6666-4666-8666-666666666666'+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)+ filesMethods.doWork.mockImplementationOnce(async (_user, taskSpace) => {+ taskSpace.task.props = {+ directories: 2,+ files: 3,+ progress: 1,+ size: 0,+ totalSize: 120+ }+ })+ const user = { id: 7 } as any+ const space = {+ inTrashRepository: true,+ realPath: '/data/users/john/trash/document',+ url: 'trash/personal/document'+ } as any++ await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')+ await flushPromises()++ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({+ status: FileTaskStatus.SUCCESS,+ props: {+ directories: 2,+ files: 3,+ progress: 100,+ size: 120,+ totalSize: 120+ }+ })+ })+it('should create a task and mark it as error when the async method fails', async () => {vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce('22222222-2222-4222-8222-222222222222')filesMethods.doWork.mockRejectedValueOnce(new Error('operation failed'))@@ -538,7 +604,7 @@ describe(FilesTasksManager.name, () => {it('should delete completed task, remove archive file and delete cache entry', async () => {const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValueOnce(undefined)- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)const user = { id: 22, tasksPath: '/tmp/tasks' } as anycacheStore.set(`${CACHE_TASK_PREFIX}-22-task-ok`, {id: 'task-ok',@@ -556,7 +622,7 @@ describe(FilesTasksManager.name, () => {it('should ignore active tasks when deleting all tasks', async () => {const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValue(undefined)+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValue(undefined)const user = { id: 31, tasksPath: '/tmp/tasks' } as anycacheStore.set(`${CACHE_TASK_PREFIX}-31-task-pending`, {id: 'task-pending',diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.tssimilarity index 56%rename from backend/src/applications/files/services/files-tasks-manager.service.tsrename to backend/src/applications/files/services/tasks/files-tasks-manager.service.tsindex 3b71c47e..5d096321 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts@@ -2,45 +2,40 @@ import { HttpException, HttpStatus, Injectable, Logger, OnModuleDestroy, Streamaimport { FastifyReply } from 'fastify'import crypto from 'node:crypto'import path from 'node:path'-import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'-import { currentTimeStamp } from '../../../common/shared'-import { Cache } from '../../../infrastructure/cache/cache.service'-import { SpaceEnv } from '../../spaces/models/space-env.model'-import { SpacesManager } from '../../spaces/services/spaces-manager.service'-import { realTrashPathFromSpace } from '../../spaces/utils/paths'-import { UserModel } from '../../users/models/user.model'-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'-import { FILE_OPERATION } from '../constants/operations'-import { CopyMoveFileDto } from '../dto/file-operations.dto'-import { FileTaskEvent } from '../events/file-events'-import type { FileTaskQueueItem } from '../interfaces/file-task-queue.interface'-import type { FileTasksPollResponse } from '../interfaces/file-task.interface'-import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'-import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'-import { SendFile } from '../utils/send-file'-import { isTaskCancellable } from '../utils/tasks'-import { FilesMethods } from './files-methods.service'+import { FastifyAuthenticatedRequest } from '../../../../authentication/interfaces/auth-request.interface'+import { currentTimeStamp } from '../../../../common/shared'+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { SpaceEnv } from '../../../spaces/models/space-env.model'+import { SpacesManager } from '../../../spaces/services/spaces-manager.service'+import { realTrashPathFromSpace } from '../../../spaces/utils/paths'+import { UserModel } from '../../../users/models/user.model'+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../../constants/cache'+import { FILE_OPERATION } from '../../constants/operations'+import { CopyMoveFileDto } from '../../dto/file-operations.dto'+import type { FileTaskQueueItem } from '../../interfaces/file-task-queue.interface'+import type { FileTasksPollResponse } from '../../interfaces/file-task.interface'+import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'+import { dirName, fileName, removeFiles } from '../../utils/files'+import { SendFile } from '../../utils/send-file'+import { isTaskCancellable } from '../../utils/tasks'+import { FilesMethods } from '../files-methods.service'import { FilesTasksQueue } from './files-tasks-queue.service'+import { FilesTasksWatcher } from './files-tasks-watcher.service'@Injectable()export class FilesTasksManager implements OnModuleDestroy {// Task cache key: `ftask-${userId}-${taskId}`.private readonly logger = new Logger(FilesTasksManager.name)private readonly watchInterval = 1000- private tasksWatcher: Record<string, NodeJS.Timeout> = {}private tasksCancellationWatcher: Record<string, NodeJS.Timeout> = {}- // Cache writes are serialized per task, while finalization blocks any late progress write.- private pendingTaskUpdates = new Map<string, Promise<void>>()- private finalizingTasks = new Set<string>()constructor(private readonly cache: Cache,private readonly filesMethods: FilesMethods,private readonly filesTasksQueue: FilesTasksQueue,+ private readonly filesTasksWatcher: FilesTasksWatcher,private readonly spacesManager: SpacesManager- ) {- FileTaskEvent.on('startWatch', this.onStartWatch)- }+ ) {}static getCacheKey(userId: number, taskId?: string): string {return `${CACHE_TASK_PREFIX}-${userId}-${taskId || '*'}`@@ -51,10 +46,6 @@ export class FilesTasksManager implements OnModuleDestroy {}onModuleDestroy(): void {- FileTaskEvent.off('startWatch', this.onStartWatch)- for (const cacheKey of Object.keys(this.tasksWatcher)) {- this.stopWatch(cacheKey)- }for (const cacheKey of Object.keys(this.tasksCancellationWatcher)) {this.stopCancellationWatch(cacheKey)}@@ -115,7 +106,7 @@ export class FilesTasksManager implements OnModuleDestroy {removeFiles(rPath).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))}// clear watcher- this.stopWatch(key)+ this.filesTasksWatcher.stopWatch(key)// remove from cachethis.cache.del(key).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))}@@ -167,12 +158,10 @@ export class FilesTasksManager implements OnModuleDestroy {}}- private async setTaskDone(cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {- this.finalizingTasks.add(cacheKey)+ private async setTaskDone(cacheKey: string, status: FileTaskStatus, result: any, finalProps?: FileTaskProps): Promise<void> {+ await this.filesTasksWatcher.beginFinalization(cacheKey)try {- this.stopWatch(cacheKey)this.stopCancellationWatch(cacheKey)- await this.pendingTaskUpdates.get(cacheKey)?.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))const task: FileTask = await this.cache.get(cacheKey)if (task) {task.status = status@@ -184,13 +173,16 @@ export class FilesTasksManager implements OnModuleDestroy {Object.assign(task, result)}}+ if (finalProps) {+ task.props = { ...task.props, ...finalProps }+ }await this.cache.set(cacheKey, task, CACHE_TASK_TTL)}await this.cache.del(cacheKey.replace(`${CACHE_TASK_PREFIX}-`, `${CACHE_TASK_CANCEL_PREFIX}-`)).catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))} finally {- this.finalizingTasks.delete(cacheKey)+ this.filesTasksWatcher.endFinalization(cacheKey)}}@@ -212,9 +204,10 @@ export class FilesTasksManager implements OnModuleDestroy {? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal): this.filesMethods[task.method](task.user, task.space, task.dto)taskPromise- .then((data: any) => {+ .then(async (data: any) => {this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })- this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) =>+ const finalProps = await this.getFinalTransferProps(task, data)+ this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data, finalProps).catch((e: Error) =>this.logger.warn({ tag: this.runTask.name, msg: `${e}` }))})@@ -230,14 +223,35 @@ export class FilesTasksManager implements OnModuleDestroy {})}- private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {+ private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any, finalProps?: FileTaskProps): Promise<void> {try {- await this.setTaskDone(cacheKey, status, result)+ await this.setTaskDone(cacheKey, status, result, finalProps)} finally {await this.filesTasksQueue.releaseAndDrain(userId)}}+ private async getFinalTransferProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {+ if (task.task.type === FILE_OPERATION.DELETE) {+ const props = task.space.task?.props+ return {+ ...props,+ ...(props?.totalSize !== undefined ? { size: props.totalSize } : {}),+ progress: 100+ }+ }+ if (task.task.type !== FILE_OPERATION.COPY && task.task.type !== FILE_OPERATION.MOVE) return+ if (!result?.path || !result?.name) return+ try {+ const publishedUrl = path.join(result.path, result.name)+ const publishedSpace = await this.spacesManager.spaceEnv(task.user, publishedUrl.split('/'))+ const props = await this.filesTasksWatcher.getPathProps(publishedSpace.realPath)+ return { ...props, progress: 100 }+ } catch (e) {+ this.logger.warn({ tag: this.getFinalTransferProps.name, msg: `${e}` })+ }+ }+private isActiveStatus(status: FileTaskStatus): boolean {return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED}@@ -265,96 +279,6 @@ export class FilesTasksManager implements OnModuleDestroy {}}- private updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {- if (this.finalizingTasks.has(cacheKey)) return Promise.resolve()- const previousUpdate = this.pendingTaskUpdates.get(cacheKey) ?? Promise.resolve()- const update = previousUpdate- .catch(() => undefined)- .then(async () => {- if (this.finalizingTasks.has(cacheKey)) return- let ftask: FileTask = await this.cache.get(cacheKey)- if (!ftask) {- this.stopWatch(cacheKey)- return- }- // Finalization may have started while the cache read was pending.- if (!this.isActiveStatus(ftask.status) || this.finalizingTasks.has(cacheKey)) {- this.stopWatch(cacheKey)- return- }- if (task) ftask = { ...ftask, ...task }- if (props) ftask.props = { ...ftask.props, ...props }- await this.cache.set(cacheKey, ftask, CACHE_TASK_TTL)- })- this.pendingTaskUpdates.set(cacheKey, update)- return update.finally(() => {- if (this.pendingTaskUpdates.get(cacheKey) === update) this.pendingTaskUpdates.delete(cacheKey)- })- }-- private startWatch(- space: SpaceEnv,- taskType: FILE_OPERATION,- rPath: string,- taskPath?: string,- taskName = fileName(rPath),- publishedPath?: string- ): void {- const taskContext = space.task- if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return- const { cacheKey, props } = taskContext- this.logger.verbose({ tag: this.startWatch.name, msg: cacheKey })- this.updateTask(cacheKey, props, {- name: taskName,- path: taskPath- }).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))- switch (taskType) {- case FILE_OPERATION.COMPRESS:- this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))- return- case FILE_OPERATION.DECOMPRESS:- this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))- return- case FILE_OPERATION.DOWNLOAD:- this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))- return- case FILE_OPERATION.COPY:- case FILE_OPERATION.DELETE:- case FILE_OPERATION.MOVE:- this.watchTask(cacheKey, () => this.updateCopyMoveTask(space, rPath))- return- default:- this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })- return- }- }-- private stopWatch(cacheKey: string): void {- if (!(cacheKey in this.tasksWatcher)) return- clearInterval(this.tasksWatcher[cacheKey])- delete this.tasksWatcher[cacheKey]- }-- private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)- }-- private watchTask(cacheKey: string, update: () => Promise<void>): void {- // Directory scans may exceed the interval, so ticks use exhaust semantics.- let updateInProgress = false- const watcher = setInterval(() => {- if (updateInProgress) return- updateInProgress = true- void update()- .catch((e: Error) => this.logger.error({ tag: this.watchTask.name, msg: `${e}` }))- .finally(() => {- updateInProgress = false- })- }, this.watchInterval)- watcher.unref()- this.tasksWatcher[cacheKey] = watcher- }-private watchCancellation(task: FileTask, userId: number, cacheKey: string): AbortController | undefined {if (!task.cancellable) returnconst controller = new AbortController()@@ -382,77 +306,4 @@ export class FilesTasksManager implements OnModuleDestroy {clearInterval(this.tasksCancellationWatcher[cacheKey])delete this.tasksCancellationWatcher[cacheKey]}-- private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)- if (size === undefined) return- space.task.props.size = size- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntries)- if (props === undefined) return- space.task.props = props- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {- try {- if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async updateCopyMoveTask(space: SpaceEnv, rPath: string): Promise<void> {- try {- if (await isPathIsDir(rPath)) {- space.task.props = await countDirEntries(rPath)- } else {- await this.calcSizeAndProgressTask(space, rPath)- }- await this.updateTask(space.task.cacheKey, space.task.props)- } catch (e) {- this.logger.error({ tag: this.updateCopyMoveTask.name, msg: `${e}` })- this.stopWatch(space.task.cacheKey)- }- }-- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)- if (size === undefined) return false- space.task.props.size = size- if (space.task.props.totalSize) {- space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize- }- return true- }-- private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {- if (!publishedPath) return read(rPath)- if (await isPathExists(rPath)) {- try {- const value = await read(rPath)- // The temporary path may be moved atomically while it is being read.- if (await isPathExists(rPath)) return value- } catch (e) {- // Retry the published path only when the temporary path disappeared during publication.- if (e?.code !== 'ENOENT') throw e- }- }- if (await isPathExists(publishedPath)) return read(publishedPath)- }}diff --git a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.tssimilarity index 96%rename from backend/src/applications/files/services/files-tasks-queue.service.spec.tsrename to backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.tsindex 3f45541d..28e67491 100644--- a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.ts@@ -1,7 +1,7 @@import { Mock } from 'vitest'-import { Cache } from '../../../infrastructure/cache/cache.service'-import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'-import type { FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'+import type { FileTaskQueueItem, FileTaskQueueStarter } from '../../interfaces/file-task-queue.interface'import { FilesTasksQueue } from './files-tasks-queue.service'describe(FilesTasksQueue.name, () => {diff --git a/backend/src/applications/files/services/files-tasks-queue.service.ts b/backend/src/applications/files/services/tasks/files-tasks-queue.service.tssimilarity index 94%rename from backend/src/applications/files/services/files-tasks-queue.service.tsrename to backend/src/applications/files/services/tasks/files-tasks-queue.service.tsindex 75f02f87..c7216c94 100644--- a/backend/src/applications/files/services/files-tasks-queue.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-queue.service.ts@@ -1,7 +1,7 @@import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'-import { Cache } from '../../../infrastructure/cache/cache.service'-import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'-import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'+import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../../interfaces/file-task-queue.interface'@Injectable()export class FilesTasksQueue implements OnModuleDestroy {diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.tsnew file mode 100644index 00000000..30884f9d--- /dev/null+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts@@ -0,0 +1,193 @@+import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'+import os from 'node:os'+import path from 'node:path'+import * as filesUtils from '../../utils/files'+import { FilesTasksTransfer } from './files-tasks-transfer.service'+import { SourceCleanupError } from '../../models/file-error'++describe(FilesTasksTransfer.name, () => {+ const cacheKey = 'ftask-7-task-id'+ let service: FilesTasksTransfer+ let tmpDir: string+ let srcPath: string+ let dstPath: string+ let stagingDir: string++ beforeEach(async () => {+ service = new FilesTasksTransfer()+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))+ srcPath = path.join(tmpDir, 'source.txt')+ dstPath = path.join(tmpDir, 'destination.txt')+ stagingDir = path.join(tmpDir, 'tasks')+ })++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true })+ vi.restoreAllMocks()+ })++ const copyAbortable = (options: Record<string, any>) => (service as any).copyAbortable(srcPath, dstPath, options)+ const moveAbortable = (options: Record<string, any>) => (service as any).moveAbortable(srcPath, dstPath, options)++ const mockCrossDevice = () => {+ vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)+ }++ it('updates task size and progress through its public copy operation', async () => {+ const content = 'content'+ const srcSpace = {+ realPath: srcPath,+ task: { cacheKey, props: {} }+ } as any+ const dstSpace = { realPath: dstPath } as any+ await writeFile(srcPath, content)++ await service.copy({ tasksPath: stagingDir } as any, srcSpace, dstSpace, false, false, false, new AbortController().signal, vi.fn())++ expect(srcSpace.task.props).toMatchObject({+ progress: 100,+ size: Buffer.byteLength(content),+ totalSize: Buffer.byteLength(content)+ })+ await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)+ })++ it('publishes a copied file only after the commit hook', async () => {+ const signal = new AbortController().signal+ let temporaryPath = ''+ let transferredBytes = 0+ await writeFile(srcPath, 'content')++ await copyAbortable({+ beforeCommit: async () => {+ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')+ },+ cacheKey,+ onTransferStart: (value: string) => {+ temporaryPath = value+ },+ onProgress: (bytes: number) => {+ transferredBytes += bytes+ },+ signal,+ stagingDir+ })++ expect(transferredBytes).toBe(Buffer.byteLength('content'))+ expect(path.dirname(temporaryPath)).toBe(stagingDir)+ expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')+ await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })+ })++ it('cleans the temporary destination when copy is cancelled', async () => {+ const controller = new AbortController()+ await writeFile(srcPath, 'content')+ controller.abort()++ await expect(copyAbortable({ cacheKey, signal: controller.signal, stagingDir })).rejects.toMatchObject({ name: 'AbortError' })++ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(readdir(stagingDir)).resolves.toEqual([])+ })++ it('copies directly to a destination on another device', async () => {+ const signal = new AbortController().signal+ let watchedPath = ''+ let destinationPrepared = false+ await writeFile(srcPath, 'content')+ mockCrossDevice()++ await copyAbortable({+ beforeCommit: async () => {+ destinationPrepared = true+ },+ cacheKey,+ onTransferStart: (value: string) => {+ expect(destinationPrepared).toBe(true)+ watchedPath = value+ },+ signal,+ stagingDir+ })++ expect(watchedPath).toBe(dstPath)+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')+ await expect(readdir(stagingDir)).resolves.toEqual([])+ })++ it('keeps an existing direct-copy destination when overwrite is rejected', async () => {+ const signal = new AbortController().signal+ await writeFile(srcPath, 'new content')+ await writeFile(dstPath, 'existing content')+ mockCrossDevice()++ await expect(copyAbortable({ cacheKey, signal, stagingDir })).rejects.toMatchObject({ code: 'EEXIST' })++ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')+ })++ it('does not prepare a direct-copy destination when already cancelled', async () => {+ const controller = new AbortController()+ const beforeCommit = vi.fn()+ await writeFile(srcPath, 'new content')+ await writeFile(dstPath, 'existing content')+ await fs.mkdir(stagingDir)+ controller.abort()+ mockCrossDevice()++ await expect(copyAbortable({ beforeCommit, cacheKey, overwrite: true, signal: controller.signal, stagingDir })).rejects.toMatchObject({+ name: 'AbortError'+ })++ expect(beforeCommit).not.toHaveBeenCalled()+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')+ })++ it('preserves a direct-copy error when destination cleanup also fails', async () => {+ const signal = new AbortController().signal+ const transferError = Object.assign(new Error('transfer failed'), { code: 'EIO' })+ await fs.mkdir(stagingDir)+ vi.spyOn(fs, 'lstat').mockImplementation(async (rPath) => {+ if (rPath === stagingDir) return { dev: 1 } as any+ throw transferError+ })+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(new Error('cleanup failed'))++ await expect(copyAbortable({ cacheKey, signal, stagingDir })).rejects.toBe(transferError)+ })++ it('copies then removes the source for a cross-device move', async () => {+ const signal = new AbortController().signal+ await writeFile(srcPath, 'content')++ await moveAbortable({ cacheKey, crossDevice: true, signal, stagingDir })++ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(readdir(stagingDir)).resolves.toEqual([])+ })++ it('reports source cleanup failure after keeping the published destination', async () => {+ const signal = new AbortController().signal+ const error = new Error('source cleanup failed')+ await writeFile(srcPath, 'content')+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)++ const cleanupError = await moveAbortable({ cacheKey, crossDevice: true, signal, stagingDir })++ expect(cleanupError).toBeInstanceOf(SourceCleanupError)+ expect(cleanupError).toMatchObject({+ cause: error,+ dstPath,+ message: 'Destination was published but the source could not be removed',+ name: SourceCleanupError.name,+ srcPath+ })+ await expect(readFile(srcPath, 'utf8')).resolves.toBe('content')+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')+ })+})diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.tsnew file mode 100644index 00000000..092f299a--- /dev/null+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts@@ -0,0 +1,259 @@+import { Injectable } from '@nestjs/common'+import { createReadStream, createWriteStream } from 'node:fs'+import fs from 'node:fs/promises'+import path from 'node:path'+import { Transform } from 'node:stream'+import { pipeline } from 'node:stream/promises'+import { UserModel } from '../../../users/models/user.model'+import { SpaceEnv } from '../../../spaces/models/space-env.model'+import { DEFAULT_HIGH_WATER_MARK } from '../../constants/files'+import { FILE_OPERATION } from '../../constants/operations'+import { FileTaskEvent } from '../../events/file-events'+import { fileSize, isPathExists, moveFiles, removeFiles } from '../../utils/files'+import { countDirEntriesAndSize, isCrossDeviceMove, taskTemporaryPath } from '../../utils/tasks'+import { FileTaskCopyTaskOptions, FileTaskTransferOptions } from '../../interfaces/file-task.interface'+import { SourceCleanupError } from '../../models/file-error'++@Injectable()+export class FilesTasksTransfer {+ async copy(+ user: UserModel,+ srcSpace: SpaceEnv,+ dstSpace: SpaceEnv,+ overwrite: boolean,+ recursive: boolean,+ isDir: boolean,+ signal: AbortSignal,+ deleteDestination: () => Promise<void>+ ): Promise<void> {+ await this.initializeTaskProps(srcSpace, isDir)+ await this.copyAbortable(srcSpace.realPath, dstSpace.realPath, {+ beforeCommit: this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination),+ cacheKey: srcSpace.task!.cacheKey,+ onProgress: (bytes) => this.updateProgress(srcSpace, bytes),+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),+ overwrite,+ recursive,+ signal,+ stagingDir: user.tasksPath+ })+ }++ async move(+ user: UserModel,+ srcSpace: SpaceEnv,+ dstSpace: SpaceEnv,+ overwrite: boolean,+ isDir: boolean,+ signal: AbortSignal | undefined,+ deleteDestination: () => Promise<void>+ ): Promise<SourceCleanupError | undefined> {+ await this.initializeTaskProps(srcSpace, isDir)+ const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)+ if (!signal) {+ // Same-device moves stay atomic and do not copy bytes.+ this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)+ await beforeCommit?.()+ await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)+ return+ }+ return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {+ beforeCommit,+ cacheKey: srcSpace.task!.cacheKey,+ crossDevice: true,+ onProgress: (bytes) => this.updateProgress(srcSpace, bytes),+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),+ overwrite,+ signal,+ stagingDir: user.tasksPath+ })+ }++ async delete(+ user: UserModel,+ space: SpaceEnv,+ trashFile: string,+ isDir: boolean,+ signal: AbortSignal | undefined,+ prepareDestination: () => Promise<void>+ ): Promise<SourceCleanupError | undefined> {+ await this.initializeTaskProps(space, isDir)+ if (!signal) {+ // Moving to trash on the same device stays atomic and does not copy bytes.+ this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)+ await prepareDestination()+ await moveFiles(space.realPath, trashFile, true)+ return+ }+ return this.moveAbortable(space.realPath, trashFile, {+ beforeCommit: prepareDestination,+ cacheKey: space.task!.cacheKey,+ crossDevice: true,+ onProgress: (bytes) => this.updateProgress(space, bytes),+ onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),+ overwrite: true,+ signal,+ stagingDir: user.tasksPath+ })+ }++ private async initializeTaskProps(space: SpaceEnv, isDir: boolean): Promise<void> {+ const metrics = isDir ? await countDirEntriesAndSize(space.realPath) : { size: await fileSize(space.realPath) }+ space.task!.props = {+ ...space.task!.props,+ ...metrics,+ progress: 1,+ size: 0,+ totalSize: metrics.size+ }+ }++ private updateProgress(space: SpaceEnv, bytes: number): void {+ const props = space.task!.props+ props.size = Math.min((props.size || 0) + bytes, props.totalSize || Number.MAX_SAFE_INTEGER)+ if (props.totalSize) {+ props.progress = Math.min((100 * props.size) / props.totalSize, 100)+ }+ }++ private prepareTaskDestination(+ srcSpace: SpaceEnv,+ dstSpace: SpaceEnv,+ overwrite: boolean,+ deleteDestination: () => Promise<void>+ ): (() => Promise<void>) | undefined {+ if (!overwrite || srcSpace.realPath.toLowerCase() === dstSpace.realPath.toLowerCase()) return+ return async () => {+ if (await isPathExists(dstSpace.realPath)) {+ await deleteDestination()+ }+ }+ }++ private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {+ FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)+ }++ private async copyAbortable(srcPath: string, dstPath: string, options: FileTaskCopyTaskOptions): Promise<void> {+ const {+ beforeCommit,+ cacheKey,+ onProgress,+ onTransferStart,+ overwrite = false,+ preserveTimestamps = true,+ recursive = true,+ signal,+ stagingDir = path.dirname(dstPath)+ } = options+ const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)+ await fs.mkdir(stagingDir, { recursive: true })+ const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)++ if (copyDirectlyToDestination) {+ let transferStarted = false+ try {+ signal.throwIfAborted()+ await beforeCommit?.()+ signal.throwIfAborted()+ await this.prepareDestination(dstPath, overwrite)+ onTransferStart?.(dstPath)+ transferStarted = true+ await this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)+ } catch (e) {+ if (transferStarted) {+ await this.cleanupAfterFailure(dstPath)+ }+ throw e+ }+ return+ }++ try {+ onTransferStart?.(temporaryPath)+ await this.copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal, onProgress)+ signal.throwIfAborted()+ await beforeCommit?.()+ await this.prepareDestination(dstPath, overwrite)+ await fs.rename(temporaryPath, dstPath)+ } catch (e) {+ await this.cleanupAfterFailure(temporaryPath)+ throw e+ }+ }++ private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {+ const { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite = false, signal, stagingDir } = options+ const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))+ if (!crossDevice) {+ signal.throwIfAborted()+ await beforeCommit?.()+ await moveFiles(srcPath, dstPath, overwrite)+ return+ }+ await this.copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite, signal, stagingDir })+ try {+ await removeFiles(srcPath)+ } catch (cause) {+ return new SourceCleanupError(srcPath, dstPath, { cause })+ }+ }++ private async copyEntry(+ srcPath: string,+ dstPath: string,+ recursive: boolean,+ preserveTimestamps: boolean,+ signal: AbortSignal,+ onProgress?: (bytes: number) => void+ ): Promise<void> {+ signal.throwIfAborted()+ const stats = await fs.lstat(srcPath)+ if (stats.isDirectory()) {+ await fs.mkdir(dstPath)+ if (recursive) {+ for (const entry of await fs.readdir(srcPath)) {+ await this.copyEntry(path.join(srcPath, entry), path.join(dstPath, entry), true, preserveTimestamps, signal, onProgress)+ }+ }+ } else if (stats.isSymbolicLink()) {+ await fs.symlink(await fs.readlink(srcPath), dstPath)+ } else {+ const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })+ const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })+ if (onProgress) {+ const progress = new Transform({+ transform(chunk: Buffer, _encoding, callback) {+ onProgress(chunk.length)+ callback(null, chunk)+ }+ })+ await pipeline(src, progress, dst, { signal })+ } else {+ await pipeline(src, dst, { signal })+ }+ }+ if (!stats.isSymbolicLink()) {+ await fs.chmod(dstPath, stats.mode)+ if (preserveTimestamps) {+ await fs.utimes(dstPath, stats.atime, stats.mtime)+ }+ }+ }++ private async prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {+ if (!(await isPathExists(dstPath))) return+ if (!overwrite) {+ throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })+ }+ await removeFiles(dstPath)+ }++ private async cleanupAfterFailure(rPath: string): Promise<void> {+ try {+ await removeFiles(rPath)+ } catch {+ // Cleanup is best-effort and must not replace the transfer error.+ }+ }+}diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.tsnew file mode 100644index 00000000..9b00d02b--- /dev/null+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts@@ -0,0 +1,181 @@+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { CACHE_TASK_TTL } from '../../constants/cache'+import { FILE_OPERATION } from '../../constants/operations'+import { FileTaskEvent } from '../../events/file-events'+import { FileTaskStatus } from '../../models/file-task'+import * as filesUtils from '../../utils/files'+import * as tasksUtils from '../../utils/tasks'+import { FilesTasksWatcher } from './files-tasks-watcher.service'+import { Mock } from 'vitest'++describe(FilesTasksWatcher.name, () => {+ let service: FilesTasksWatcher+ let cacheStore: Map<string, any>+ let cache: {+ get: Mock+ set: Mock+ }++ const flushPromises = async () => {+ await Promise.resolve()+ await Promise.resolve()+ }++ const makeSpace = (cacheKey = 'task-1', props: Record<string, any> = {}) =>+ ({+ url: 'files/personal/source.txt',+ task: { cacheKey, props }+ }) as any++ beforeEach(() => {+ cacheStore = new Map()+ cache = {+ get: vi.fn(async (key: string) => cacheStore.get(key)),+ set: vi.fn(async (key: string, value: unknown) => {+ cacheStore.set(key, value)+ return true+ })+ }+ service = new FilesTasksWatcher(cache as unknown as Cache)+ })++ afterEach(() => {+ service.onModuleDestroy()+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it('subscribes to task watch events and unsubscribes on module destroy', () => {+ const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')++ service.onModuleDestroy()++ expect(FileTaskEvent.listenerCount('startWatch')).toBe(listenersBeforeDestroy - 1)+ })++ it('returns file or directory metrics for a watched path', async () => {+ vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false).mockResolvedValueOnce(true)+ vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)+ vi.spyOn(tasksUtils, 'countDirEntriesAndSize').mockResolvedValueOnce({ files: 2, directories: 1, size: 84 })++ await expect(service.getPathProps('/files/report.txt')).resolves.toEqual({ size: 42 })+ await expect(service.getPathProps('/files/archive')).resolves.toEqual({ files: 2, directories: 1, size: 84 })+ })++ it('starts a download watcher with the published task metadata', async () => {+ const space = makeSpace()+ const updateTask = vi.spyOn(service as any, 'updateTask').mockResolvedValue(undefined)+ const updateDownloadTask = vi.spyOn(service as any, 'updateDownloadTask').mockResolvedValue(undefined)+ const watchTask = vi.spyOn(service as any, 'watchTask').mockImplementation((_cacheKey: string, update: () => Promise<void>) => void update())++ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, '/files/report.txt', '/tmp/report.part')+ await flushPromises()++ expect(updateTask).toHaveBeenCalledWith('task-1', {}, { name: 'report.txt', path: 'files/personal' })+ expect(watchTask).toHaveBeenCalledWith('task-1', expect.any(Function))+ expect(updateDownloadTask).toHaveBeenCalledWith(space, '/tmp/report.part', '/files/report.txt')+ })++ it('does not start duplicate watchers for the same task', () => {+ const space = makeSpace()+ const watchTask = vi.spyOn(service as any, 'watchTask')+ ;(service as any).tasksWatcher['task-1'] = setInterval(() => undefined, 1000)+ ;(service as any).startWatch(space, FILE_OPERATION.COPY, '/files/report.txt')++ expect(watchTask).not.toHaveBeenCalled()+ })++ it('does not overlap interval updates', async () => {+ vi.useFakeTimers()+ let finishUpdate!: () => void+ const update = vi.fn(+ () =>+ new Promise<void>((resolve) => {+ finishUpdate = resolve+ })+ )++ ;(service as any).watchTask('task-1', update)+ await vi.advanceTimersByTimeAsync(2000)++ expect(update).toHaveBeenCalledTimes(1)++ finishUpdate()+ await flushPromises()+ await vi.advanceTimersByTimeAsync(1000)++ expect(update).toHaveBeenCalledTimes(2)+ })++ it('merges active task metadata and properties in the cache', async () => {+ cacheStore.set('task-1', {+ name: 'source.txt',+ path: 'files/personal',+ props: { progress: 1, totalSize: 100 },+ status: FileTaskStatus.PENDING+ })++ await (service as any).updateTask('task-1', { progress: 40, size: 40 }, { name: 'destination.txt' })++ expect(cache.set).toHaveBeenCalledWith(+ 'task-1',+ {+ name: 'destination.txt',+ path: 'files/personal',+ props: { progress: 40, size: 40, totalSize: 100 },+ status: FileTaskStatus.PENDING+ },+ CACHE_TASK_TTL+ )+ })++ it.each([undefined, FileTaskStatus.SUCCESS])('stops watching when the cached task is missing or inactive', async (status) => {+ if (status !== undefined) {+ cacheStore.set('task-1', { props: {}, status })+ }+ const stopWatch = vi.spyOn(service, 'stopWatch')++ await (service as any).updateTask('task-1', { size: 10 })++ expect(stopWatch).toHaveBeenCalledWith('task-1')+ expect(cache.set).not.toHaveBeenCalled()+ })++ it('waits for pending updates and prevents cache writes during finalization', async () => {+ let resolveGet!: (value: unknown) => void+ cache.get.mockReturnValueOnce(+ new Promise((resolve) => {+ resolveGet = resolve+ })+ )+ const update = (service as any).updateTask('task-1', { size: 10 })+ await flushPromises()++ const finalization = service.beginFinalization('task-1')+ resolveGet({ props: {}, status: FileTaskStatus.PENDING })+ await Promise.all([update, finalization])++ expect(cache.set).not.toHaveBeenCalled()+ expect((service as any).finalizingTasks.has('task-1')).toBe(true)++ service.endFinalization('task-1')+ expect((service as any).finalizingTasks.has('task-1')).toBe(false)+ })++ it('falls back to the published file and updates download progress', async () => {+ const space = makeSpace('task-1', { size: 0, totalSize: 200 })+ cacheStore.set('task-1', {+ props: { size: 0, totalSize: 200 },+ status: FileTaskStatus.PENDING+ })+ vi.spyOn(filesUtils, 'isPathExists').mockImplementation(async (rPath: string) => rPath === '/files/report.txt')+ vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false)+ vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(50)++ await (service as any).updateDownloadTask(space, '/tmp/report.part', '/files/report.txt')++ expect(filesUtils.fileSize).toHaveBeenCalledWith('/files/report.txt')+ expect(space.task.props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })+ expect(cacheStore.get('task-1').props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })+ })+})diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.tsnew file mode 100644index 00000000..16d09d1d--- /dev/null+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts@@ -0,0 +1,201 @@+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'+import { Cache } from '../../../../infrastructure/cache/cache.service'+import { SpaceEnv } from '../../../spaces/models/space-env.model'+import { CACHE_TASK_TTL } from '../../constants/cache'+import { FILE_OPERATION } from '../../constants/operations'+import { FileTaskEvent } from '../../events/file-events'+import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'+import { dirName, fileName, fileSize, isPathExists, isPathIsDir } from '../../utils/files'+import { countDirEntriesAndSize } from '../../utils/tasks'++@Injectable()+export class FilesTasksWatcher implements OnModuleDestroy {+ private readonly logger = new Logger(FilesTasksWatcher.name)+ private readonly watchInterval = 1000+ private tasksWatcher: Record<string, NodeJS.Timeout> = {}+ private pendingTaskUpdates = new Map<string, Promise<void>>()+ private finalizingTasks = new Set<string>()++ constructor(private readonly cache: Cache) {+ FileTaskEvent.on('startWatch', this.onStartWatch)+ }++ onModuleDestroy(): void {+ FileTaskEvent.off('startWatch', this.onStartWatch)+ for (const cacheKey of Object.keys(this.tasksWatcher)) {+ this.stopWatch(cacheKey)+ }+ }++ async beginFinalization(cacheKey: string): Promise<void> {+ this.finalizingTasks.add(cacheKey)+ this.stopWatch(cacheKey)+ await this.pendingTaskUpdates.get(cacheKey)?.catch((e: Error) => this.logger.error({ tag: this.beginFinalization.name, msg: `${e}` }))+ }++ endFinalization(cacheKey: string): void {+ this.finalizingTasks.delete(cacheKey)+ }++ stopWatch(cacheKey: string): void {+ if (!(cacheKey in this.tasksWatcher)) return+ clearInterval(this.tasksWatcher[cacheKey])+ delete this.tasksWatcher[cacheKey]+ }++ async getPathProps(rPath: string): Promise<Pick<FileTaskProps, 'files' | 'directories' | 'size'>> {+ return (await isPathIsDir(rPath)) ? countDirEntriesAndSize(rPath) : { size: await fileSize(rPath) }+ }++ private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)+ }++ private startWatch(+ space: SpaceEnv,+ taskType: FILE_OPERATION,+ rPath: string,+ taskPath?: string,+ taskName = fileName(rPath),+ publishedPath?: string+ ): void {+ const taskContext = space.task+ if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return+ const { cacheKey, props } = taskContext+ this.updateTask(cacheKey, props, {+ name: taskName,+ path: taskPath+ }).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))+ switch (taskType) {+ case FILE_OPERATION.COMPRESS:+ this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))+ return+ case FILE_OPERATION.DECOMPRESS:+ this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))+ return+ case FILE_OPERATION.DOWNLOAD:+ this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))+ return+ case FILE_OPERATION.COPY:+ case FILE_OPERATION.DELETE:+ case FILE_OPERATION.MOVE:+ this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))+ return+ default:+ this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })+ }+ }++ private watchTask(cacheKey: string, update: () => Promise<void>): void {+ // Directory scans may exceed the interval, so ticks use exhaust semantics.+ let updateInProgress = false+ const watcher = setInterval(() => {+ if (updateInProgress) return+ updateInProgress = true+ void update()+ .catch((e: Error) => this.logger.error({ tag: this.watchTask.name, msg: `${e}` }))+ .finally(() => {+ updateInProgress = false+ })+ }, this.watchInterval)+ watcher.unref()+ this.tasksWatcher[cacheKey] = watcher+ }++ private updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {+ if (this.finalizingTasks.has(cacheKey)) return Promise.resolve()+ const previousUpdate = this.pendingTaskUpdates.get(cacheKey) ?? Promise.resolve()+ const update = previousUpdate+ .catch(() => undefined)+ .then(async () => {+ if (this.finalizingTasks.has(cacheKey)) return+ let fileTask: FileTask = await this.cache.get(cacheKey)+ if (!fileTask) {+ this.stopWatch(cacheKey)+ return+ }+ if (!this.isActiveStatus(fileTask.status) || this.finalizingTasks.has(cacheKey)) {+ this.stopWatch(cacheKey)+ return+ }+ if (task) fileTask = { ...fileTask, ...task }+ if (props) fileTask.props = { ...fileTask.props, ...props }+ await this.cache.set(cacheKey, fileTask, CACHE_TASK_TTL)+ })+ this.pendingTaskUpdates.set(cacheKey, update)+ return update.finally(() => {+ if (this.pendingTaskUpdates.get(cacheKey) === update) this.pendingTaskUpdates.delete(cacheKey)+ })+ }++ private isActiveStatus(status: FileTaskStatus): boolean {+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED+ }++ private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {+ try {+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)+ if (size === undefined) return+ space.task.props.size = size+ await this.updateTask(space.task.cacheKey, space.task.props)+ } catch (e) {+ this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })+ this.stopWatch(space.task.cacheKey)+ }+ }++ private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {+ try {+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)+ if (props === undefined) return+ space.task.props = props+ await this.updateTask(space.task.cacheKey, space.task.props)+ } catch (e) {+ this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })+ this.stopWatch(space.task.cacheKey)+ }+ }++ private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {+ try {+ if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return+ await this.updateTask(space.task.cacheKey, space.task.props)+ } catch (e) {+ this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })+ this.stopWatch(space.task.cacheKey)+ }+ }++ private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {+ const isDir = await this.readWatchedPath(rPath, publishedPath, isPathIsDir)+ if (isDir === undefined) return false+ if (isDir) {+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)+ if (props === undefined) return false+ space.task.props = { ...space.task.props, ...props }+ } else {+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)+ if (size === undefined) return false+ space.task.props.size = size+ }+ if (space.task.props.totalSize) {+ space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize+ }+ return true+ }++ private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {+ if (!publishedPath) return read(rPath)+ if (await isPathExists(rPath)) {+ try {+ const value = await read(rPath)+ // The temporary path may be moved atomically while it is being read.+ if (await isPathExists(rPath)) return value+ } catch (e) {+ // Retry the published path only when the temporary path disappeared during publication.+ if (e?.code !== 'ENOENT') throw e+ }+ }+ if (await isPathExists(publishedPath)) return read(publishedPath)+ }+}diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex 6ad551c9..a3841ec0 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -244,7 +244,7 @@ export function copyFileContent(srcPath: string, dstPath: string): Promise<void>return writeFromStream(dstPath, srcStream)}-async function walkDir(+export async function walkDir(rPath: string,onEntry: (entry: Dirent, entryPath: string) => Promise<void> | void,errors?: Record<string, string>@@ -291,25 +291,6 @@ export async function dirListFileNames(rPath: string): Promise<string[]> {return (await fs.readdir(rPath)).map((path: string) => fileName(path))}-export async function countDirEntries(rPath: string): Promise<{ files: number; directories: number }> {- const entriesCount = { files: 0, directories: 0 }- const ignoredErrors: Record<string, string> = {}-- await walkDir(- rPath,- (entry: Dirent) => {- if (entry.isDirectory()) {- entriesCount.directories++- } else {- entriesCount.files++- }- },- ignoredErrors- )-- return entriesCount-}-export async function dirHasChildren(rPath: string, mustContainsDirs = true): Promise<boolean> {for await (const file of await fs.opendir(rPath)) {if (mustContainsDirs) {diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.tsindex 71f30d77..dbd566c8 100644--- a/backend/src/applications/files/utils/tasks.spec.ts+++ b/backend/src/applications/files/utils/tasks.spec.ts@@ -1,22 +1,19 @@-import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'+import fs, { mkdtemp, rm, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { FILE_OPERATION } from '../constants/operations'-import * as filesUtils from './files'-import { copyAbortable, createTaskTemporaryDir, isTaskCancellable, moveAbortable, SourceCleanupError, taskTemporaryPath } from './tasks'+import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath } from './tasks'-describe('abortable file tasks', () => {+describe('file task utilities', () => {const cacheKey = 'ftask-7-task-id'let tmpDir: stringlet srcPath: stringlet dstPath: string- let stagingDir: stringbeforeEach(async () => {tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))srcPath = path.join(tmpDir, 'source.txt')dstPath = path.join(tmpDir, 'destination.txt')- stagingDir = path.join(tmpDir, 'tasks')})afterEach(async () => {@@ -24,11 +21,6 @@ describe('abortable file tasks', () => {vi.restoreAllMocks()})- const mockCrossDevice = () => {- vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)- vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)- }-it('marks intrinsically abortable operations as cancellable', async () => {await expect(isTaskCancellable(FILE_OPERATION.COPY, '/source')).resolves.toBe(true)await expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD, '/source')).resolves.toBe(true)@@ -57,139 +49,4 @@ describe('abortable file tasks', () => {expect(path.basename(directoryPath)).toBe(`.${cacheKey}-archive`)expect((await fs.stat(directoryPath)).isDirectory()).toBe(true)})-- it('publishes a copied file only after the commit hook', async () => {- const signal = new AbortController().signal- let temporaryPath = ''- await writeFile(srcPath, 'content')-- await copyAbortable(srcPath, dstPath, {- beforeCommit: async () => {- await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })- await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')- },- cacheKey,- onTransferStart: (value) => {- temporaryPath = value- },- signal,- stagingDir- })-- expect(path.dirname(temporaryPath)).toBe(stagingDir)- expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')- await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })- })-- it('cleans the temporary destination when copy is cancelled', async () => {- const controller = new AbortController()- await writeFile(srcPath, 'content')- controller.abort()-- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal: controller.signal, stagingDir })).rejects.toMatchObject({- name: 'AbortError'- })-- await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })- await expect(readdir(stagingDir)).resolves.toEqual([])- })-- it('copies directly to a destination on another device', async () => {- const signal = new AbortController().signal- let watchedPath = ''- let destinationPrepared = false- await writeFile(srcPath, 'content')- mockCrossDevice()-- await copyAbortable(srcPath, dstPath, {- beforeCommit: async () => {- destinationPrepared = true- },- cacheKey,- onTransferStart: (value) => {- expect(destinationPrepared).toBe(true)- watchedPath = value- },- signal,- stagingDir- })-- expect(watchedPath).toBe(dstPath)- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')- await expect(readdir(stagingDir)).resolves.toEqual([])- })-- it('keeps an existing direct-copy destination when overwrite is rejected', async () => {- const signal = new AbortController().signal- await writeFile(srcPath, 'new content')- await writeFile(dstPath, 'existing content')- mockCrossDevice()-- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toMatchObject({ code: 'EEXIST' })-- await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')- })-- it('does not prepare a direct-copy destination when already cancelled', async () => {- const controller = new AbortController()- const beforeCommit = vi.fn()- await writeFile(srcPath, 'new content')- await writeFile(dstPath, 'existing content')- await fs.mkdir(stagingDir)- controller.abort()- mockCrossDevice()-- await expect(- copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, overwrite: true, signal: controller.signal, stagingDir })- ).rejects.toMatchObject({ name: 'AbortError' })-- expect(beforeCommit).not.toHaveBeenCalled()- await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')- })-- it('preserves a direct-copy error when destination cleanup also fails', async () => {- const signal = new AbortController().signal- const transferError = Object.assign(new Error('transfer failed'), { code: 'EIO' })- await fs.mkdir(stagingDir)- vi.spyOn(fs, 'lstat').mockImplementation(async (rPath) => {- if (rPath === stagingDir) return { dev: 1 } as any- throw transferError- })- vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)- vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(new Error('cleanup failed'))-- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toBe(transferError)- })-- it('copies then removes the source for a cross-device move', async () => {- const signal = new AbortController().signal- await writeFile(srcPath, 'content')-- await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })-- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')- await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })- await expect(readdir(stagingDir)).resolves.toEqual([])- })-- it('reports source cleanup failure after keeping the published destination', async () => {- const signal = new AbortController().signal- const error = new Error('source cleanup failed')- await writeFile(srcPath, 'content')- vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)-- const cleanupError = await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })-- expect(cleanupError).toBeInstanceOf(SourceCleanupError)- expect(cleanupError).toMatchObject({- cause: error,- dstPath,- message: 'Destination was published but the source could not be removed',- name: SourceCleanupError.name,- srcPath- })- await expect(readFile(srcPath, 'utf8')).resolves.toBe('content')- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')- })})diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.tsindex b38748e8..e159c448 100644--- a/backend/src/applications/files/utils/tasks.ts+++ b/backend/src/applications/files/utils/tasks.ts@@ -1,42 +1,9 @@-import { createReadStream, createWriteStream } from 'node:fs'+import { Dirent } from 'node:fs'import fs from 'node:fs/promises'import path from 'node:path'-import { pipeline } from 'node:stream/promises'-import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'import { FILE_OPERATION } from '../constants/operations'-import { isPathExists, moveFiles, removeFiles } from './files'--export interface TransferOptions {- // Runs immediately before the destination can be modified or published.- beforeCommit?: () => Promise<void>- // Identifies the owning task so cleanup can preserve its staging entry.- cacheKey: string- // Allows callers that already resolved the filesystem topology to skip another stat.- crossDevice?: boolean- // Provides the path whose progress must be watched during the transfer.- onTransferStart?: (temporaryPath: string) => void- overwrite?: boolean- signal: AbortSignal- // Preferred staging directory. Atomic publication requires it to share the destination filesystem.- stagingDir?: string-}--export interface CopyTaskOptions extends TransferOptions {- preserveTimestamps?: boolean- recursive?: boolean-}--// The destination is committed, but the obsolete source still requires manual or deferred cleanup.-export class SourceCleanupError extends Error {- constructor(- readonly srcPath: string,- readonly dstPath: string,- options: ErrorOptions- ) {- super('Destination was published but the source could not be removed', options)- this.name = SourceCleanupError.name- }-}+import { isPathExists, walkDir } from './files'+import type { FileTaskProps } from '../models/file-task'export function taskTemporaryPrefix(cacheKey: string): string {return `.${cacheKey}-`@@ -53,33 +20,6 @@ export async function createTaskTemporaryDir(parentPath: string, cacheKey: strinreturn temporaryPath}-async function copyEntry(srcPath: string, dstPath: string, recursive: boolean, preserveTimestamps: boolean, signal: AbortSignal): Promise<void> {- signal.throwIfAborted()- const stats = await fs.lstat(srcPath)- if (stats.isDirectory()) {- await fs.mkdir(dstPath)- if (recursive) {- for (const entry of await fs.readdir(srcPath)) {- await copyEntry(path.join(srcPath, entry), path.join(dstPath, entry), true, preserveTimestamps, signal)- }- }- } else if (stats.isSymbolicLink()) {- await fs.symlink(await fs.readlink(srcPath), dstPath)- } else {- await pipeline(- createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK }),- createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK }),- { signal }- )- }- if (!stats.isSymbolicLink()) {- await fs.chmod(dstPath, stats.mode)- if (preserveTimestamps) {- await fs.utimes(dstPath, stats.atime, stats.mtime)- }- }-}-async function existingParentPath(dstPath: string): Promise<string> {let parentPath = path.dirname(dstPath)while (!(await isPathExists(parentPath))) {@@ -115,89 +55,27 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, d}}-async function prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {- if (!(await isPathExists(dstPath))) return- if (!overwrite) {- throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })- }- await removeFiles(dstPath)-}--async function cleanupAfterFailure(rPath: string): Promise<void> {- try {- await removeFiles(rPath)- } catch {- // Cleanup is best-effort and must not replace the transfer error.- }-}--export async function copyAbortable(srcPath: string, dstPath: string, options: CopyTaskOptions): Promise<void> {- const {- beforeCommit,- cacheKey,- onTransferStart,- overwrite = false,- preserveTimestamps = true,- recursive = true,- signal,- stagingDir = path.dirname(dstPath)- } = options- const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)- await fs.mkdir(stagingDir, { recursive: true })- const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)-- // A staging file cannot be renamed atomically across filesystems. In that case,- // copy directly to the destination and remove partial data on handled failures.- // beforeCommit may already have moved an overwritten destination to trash when- // cancellation occurs; this recoverable side effect is accepted for external filesystems.- if (copyDirectlyToDestination) {- let transferStarted = false- try {- // Check before modifying the destination, then again before starting the long direct transfer.- signal.throwIfAborted()- await beforeCommit?.()- signal.throwIfAborted()- await prepareDestination(dstPath, overwrite)- onTransferStart?.(dstPath)- transferStarted = true- await copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal)- } catch (e) {- if (transferStarted) {- await cleanupAfterFailure(dstPath)+export async function countDirEntriesAndSize(rPath: string): Promise<Pick<FileTaskProps, 'files' | 'directories' | 'size'>> {+ const entriesCount = { files: 0, directories: 0, size: 0 }+ const ignoredErrors: Record<string, string> = {}++ await walkDir(+ rPath,+ async (entry: Dirent, entryPath: string) => {+ if (entry.isDirectory()) {+ entriesCount.directories+++ } else {+ entriesCount.files+++ if (!entry.isFile()) return+ try {+ entriesCount.size += (await fs.stat(entryPath)).size+ } catch {+ // ignore+ }}- throw e- }- return- }-- // Same-filesystem staging keeps the destination untouched until the final rename.- try {- onTransferStart?.(temporaryPath)- await copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal)- signal.throwIfAborted()- await beforeCommit?.()- await prepareDestination(dstPath, overwrite)- await fs.rename(temporaryPath, dstPath)- } catch (e) {- await cleanupAfterFailure(temporaryPath)- throw e- }-}+ },+ ignoredErrors+ )-export async function moveAbortable(srcPath: string, dstPath: string, options: TransferOptions): Promise<SourceCleanupError | undefined> {- const { beforeCommit, cacheKey, onTransferStart, overwrite = false, signal, stagingDir } = options- const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))- if (!crossDevice) {- signal.throwIfAborted()- await beforeCommit?.()- await moveFiles(srcPath, dstPath, overwrite)- return- }- await copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, onTransferStart, overwrite, signal, stagingDir })- try {- await removeFiles(srcPath)- } catch (cause) {- // Publication is the commit point. The caller must update its persistent state before reporting this cleanup failure.- return new SourceCleanupError(srcPath, dstPath, { cause })- }+ return entriesCount}