Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): improve task progress tracking for copy and move operations

johaven79394910fd94
diff --git a/backend/src/applications/files/files-tasks.controller.spec.ts b/backend/src/applications/files/files-tasks.controller.spec.ts
index 0b49579d..2c95f7fc 100644
--- a/backend/src/applications/files/files-tasks.controller.spec.ts
+++ b/backend/src/applications/files/files-tasks.controller.spec.ts
@@ -2,7 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing'
import { Cache } from '../../infrastructure/cache/cache.service'
import { FilesTasksController } from './files-tasks.controller'
import { FilesMethods } from './services/files-methods.service'
-import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
import { Mock } from 'vitest'
describe(FilesTasksController.name, () => {
diff --git a/backend/src/applications/files/files-tasks.controller.ts b/backend/src/applications/files/files-tasks.controller.ts
index 862ac07f..fb1c311b 100644
--- a/backend/src/applications/files/files-tasks.controller.ts
+++ b/backend/src/applications/files/files-tasks.controller.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../users/models/user.model'
import { API_FILES_TASKS, FILES_ROUTE } from './constants/routes'
import { FileTasksPollDto } from './dto/file-tasks-poll.dto'
import type { FileTasksPollResponse } from './interfaces/file-task.interface'
-import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
@Controller(API_FILES_TASKS)
export class FilesTasksController {
diff --git a/backend/src/applications/files/files.controller.spec.ts b/backend/src/applications/files/files.controller.spec.ts
index 9d2dd7ee..665d5a95 100644
--- a/backend/src/applications/files/files.controller.spec.ts
+++ b/backend/src/applications/files/files.controller.spec.ts
@@ -7,7 +7,7 @@ import { FilesController } from './files.controller'
import { FilesMethods } from './services/files-methods.service'
import { FilesRecents } from './services/files-recents.service'
import { FilesSearchManager } from './services/files-search-manager.service'
-import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
import { FilesContentIndexer } from './services/files-content-indexer.service'
describe(FilesController.name, () => {
diff --git a/backend/src/applications/files/files.controller.ts b/backend/src/applications/files/files.controller.ts
index 72ec125a..8764d7fd 100644
--- a/backend/src/applications/files/files.controller.ts
+++ b/backend/src/applications/files/files.controller.ts
@@ -43,7 +43,7 @@ import { FileRecent } from './schemas/file-recent.interface'
import { FilesMethods } from './services/files-methods.service'
import { FilesRecents } from './services/files-recents.service'
import { FilesSearchManager } from './services/files-search-manager.service'
-import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
import { FilesContentIndexer } from './services/files-content-indexer.service'
import { UserHaveRole } from '../users/decorators/roles.decorator'
import { USER_ROLE } from '../users/constants/user'
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index 919c2628..5397c78e 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -15,8 +15,10 @@ import { FilesQueries } from './services/files-queries.service'
import { FilesRecents } from './services/files-recents.service'
import { FilesScheduler } from './services/files-scheduler.service'
import { FilesSearchManager } from './services/files-search-manager.service'
-import { FilesTasksManager } from './services/files-tasks-manager.service'
-import { FilesTasksQueue } from './services/files-tasks-queue.service'
+import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
+import { FilesTasksQueue } from './services/tasks/files-tasks-queue.service'
+import { FilesTasksTransfer } from './services/tasks/files-tasks-transfer.service'
+import { FilesTasksWatcher } from './services/tasks/files-tasks-watcher.service'
import { FilesEventManager } from './services/files-event-manager.service'
import { FilesQuotaManager } from './services/files-quota-manager.service'
import { FilesTrashRetention } from './services/files-trash-retention.service'
@@ -34,6 +36,8 @@ import { FilesTrashRetention } from './services/files-trash-retention.service'
FilesLockManager,
FilesTasksManager,
FilesTasksQueue,
+ FilesTasksTransfer,
+ FilesTasksWatcher,
FilesScheduler,
FilesRecents,
FilesContentParser,
diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.ts
index 6cdb7c1e..82b5efc9 100644
--- a/backend/src/applications/files/interfaces/file-task.interface.ts
+++ b/backend/src/applications/files/interfaces/file-task.interface.ts
@@ -5,3 +5,18 @@ export interface FileTasksPollResponse {
ended: FileTask[]
missingIds: string[]
}
+export interface FileTaskTransferOptions {
+ beforeCommit?: () => Promise<void>
+ cacheKey: string
+ crossDevice?: boolean
+ onTransferStart?: (temporaryPath: string) => void
+ onProgress?: (bytes: number) => void
+ overwrite?: boolean
+ signal: AbortSignal
+ stagingDir?: string
+}
+
+export interface FileTaskCopyTaskOptions extends FileTaskTransferOptions {
+ preserveTimestamps?: boolean
+ recursive?: boolean
+}
diff --git a/backend/src/applications/files/models/file-error.ts b/backend/src/applications/files/models/file-error.ts
index 0df437d1..0cfcf61f 100644
--- a/backend/src/applications/files/models/file-error.ts
+++ b/backend/src/applications/files/models/file-error.ts
@@ -7,3 +7,15 @@ export class FileError extends Error {
this.httpCode = httpCode
}
}
+
+export class SourceCleanupError extends Error {
+ // The destination is committed, but the obsolete source still requires manual or deferred cleanup.
+ constructor(
+ readonly srcPath: string,
+ readonly dstPath: string,
+ options: ErrorOptions
+ ) {
+ super('Destination was published but the source could not be removed', options)
+ this.name = SourceCleanupError.name
+ }
+}
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index abee5c11..a80f6b00 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -18,7 +18,7 @@ import { ACTION } from '../../../common/constants'
import { FILE_OPERATION } from '../constants/operations'
import { DownloadFileDto } from '../dto/file-operations.dto'
import { FileEvent, FileTaskEvent } from '../events/file-events'
-import { FileError } from '../models/file-error'
+import { FileError, SourceCleanupError } from '../models/file-error'
import { LockConflict } from '../models/file-lock-error'
import { FILE_ERROR_MESSAGES } from '../utils/errors'
import { SendFile } from '../utils/send-file'
@@ -29,6 +29,7 @@ import * as taskUtils from '../utils/tasks'
import { FilesLockManager } from './files-lock-manager.service'
import { FilesManager } from './files-manager.service'
import { FilesQueries } from './files-queries.service'
+import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'
import { Mock } from 'vitest'
vi.mock('archiver', () => ({
@@ -42,6 +43,7 @@ vi.mock('node:dns/promises', () => ({
describe(FilesManager.name, () => {
let service: FilesManager
+ let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock }
let http: { axiosRef: Mock }
const lookupMock = lookup as Mock
let filesQueries: { moveFiles: Mock; deleteFiles: Mock }
@@ -127,8 +129,9 @@ describe(FilesManager.name, () => {
expect(filesUtils.copyFiles).not.toHaveBeenCalled()
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.delete).not.toHaveBeenCalled()
expect(filesLockManager.create).not.toHaveBeenCalled()
expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()
expect(filesLockManager.checkConflicts).not.toHaveBeenCalled()
@@ -150,6 +153,27 @@ describe(FilesManager.name, () => {
notificationsManager = {
create: vi.fn().mockResolvedValue(undefined)
}
+ filesTasksTransfer = {
+ copy: vi
+ .fn()
+ .mockImplementation(
+ async (
+ _user: any,
+ srcSpace: any,
+ _dstSpace: any,
+ overwrite: boolean,
+ _recursive: boolean,
+ _isDir: boolean,
+ _signal: AbortSignal,
+ deleteDestination: () => Promise<void>
+ ) => {
+ srcSpace.task.props = { ...srcSpace.task.props, progress: 40, size: 40, totalSize: 100 }
+ if (overwrite) await deleteDestination()
+ }
+ ),
+ move: vi.fn().mockResolvedValue(undefined),
+ delete: vi.fn().mockResolvedValue(undefined)
+ }
filesLockManager = {
create: vi.fn().mockResolvedValue([true, { key: 'lock-1' }]),
checkConflicts: vi.fn().mockResolvedValue(undefined),
@@ -168,6 +192,7 @@ describe(FilesManager.name, () => {
{ provide: NotificationsManager, useValue: notificationsManager },
{ provide: HttpService, useValue: http },
{ provide: FilesLockManager, useValue: filesLockManager },
+ { provide: FilesTasksTransfer, useValue: filesTasksTransfer },
FilesManager
]
}).compile()
@@ -192,16 +217,7 @@ describe(FilesManager.name, () => {
vi.spyOn(filesUtils, 'dirSize').mockResolvedValue([123, {}] as any)
vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/unique-path.txt')
vi.spyOn(filesUtils, 'uniqueDatedFilePath').mockResolvedValue({ isDir: false, path: '/trash/file-2026.txt' })
- vi.spyOn(taskUtils, 'copyAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {
- options.onTransferStart?.('/tmp/task-transfer')
- await options.beforeCommit?.()
- })
vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue('/data/users/john/tmp/tasks/.task-d-archive')
- vi.spyOn(taskUtils, 'moveAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {
- options.onTransferStart?.('/tmp/task-transfer')
- await options.beforeCommit?.()
- return undefined
- })
vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>
path.join(parentPath, `.${cacheKey}-${path.basename(name)}`)
)
@@ -812,7 +828,7 @@ describe(FilesManager.name, () => {
expect(filesLockManager.checkConflicts).toHaveBeenCalledWith(dst.dbFile, DEPTH.RESOURCE, { userId: 7, lockTokens: undefined })
expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false, false)
- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: dst.realPath })
})
@@ -846,7 +862,7 @@ describe(FilesManager.name, () => {
await service.copyMove(user, src, dst, true)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/src-base/src.txt', '/dst-base/dst.txt', false)
- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()
expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: src, action: ACTION.DELETE_PERMANENTLY, rPath: '/src-base/src.txt' })
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: '/dst-base/dst.txt' })
@@ -865,22 +881,13 @@ describe(FilesManager.name, () => {
dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }
})
const signal = new AbortController().signal
- const cleanupError = new taskUtils.SourceCleanupError(src.realPath, dst.realPath, { cause: new Error('cleanup failed') })
+ const cleanupError = new SourceCleanupError(src.realPath, dst.realPath, { cause: new Error('cleanup failed') })
prepareFileTransfer(src.realPath, dst.realPath)
- vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)
+ filesTasksTransfer.move.mockResolvedValueOnce(cleanupError)
await expect(service.copyMove(user, src, dst, true, false, false, undefined, signal)).rejects.toBe(cleanupError)
- expect(taskUtils.moveAbortable).toHaveBeenCalledWith(
- src.realPath,
- dst.realPath,
- expect.objectContaining({
- cacheKey: 'task-move',
- crossDevice: true,
- signal,
- stagingDir: user.tasksPath
- })
- )
+ expect(filesTasksTransfer.move).toHaveBeenCalledWith(user, src, dst, false, false, signal, expect.any(Function))
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)
})
@@ -900,7 +907,7 @@ describe(FilesManager.name, () => {
await service.copyMove(user, src, dst, true)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false)
- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.move).not.toHaveBeenCalled()
})
it('should preserve the regular overwrite path outside a task context', async () => {
@@ -919,7 +926,7 @@ describe(FilesManager.name, () => {
expect(deleteSpy).toHaveBeenCalledWith(user, dst)
expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, true, false)
- expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.copy).not.toHaveBeenCalled()
expect(deleteSpy.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.copyFiles).mock.invocationCallOrder[0])
})
@@ -939,18 +946,9 @@ describe(FilesManager.name, () => {
await service.copyMove(user, src, dst, false, true, false, undefined, signal)
- expect(taskUtils.copyAbortable).toHaveBeenCalledWith(
- src.realPath,
- dst.realPath,
- expect.objectContaining({
- beforeCommit: expect.any(Function),
- cacheKey: 'task-copy',
- overwrite: true,
- signal,
- stagingDir: user.tasksPath
- })
- )
+ expect(filesTasksTransfer.copy).toHaveBeenCalledWith(user, src, dst, true, false, false, signal, expect.any(Function))
expect(deleteSpy).toHaveBeenCalledWith(user, dst)
+ expect(src.task.props).toMatchObject({ progress: 40, size: 40, totalSize: 100 })
})
})
@@ -991,23 +989,13 @@ describe(FilesManager.name, () => {
})
const trashFile = '/data/users/john/trash/documents/document.txt'
const signal = new AbortController().signal
- const cleanupError = new taskUtils.SourceCleanupError(space.realPath, trashFile, { cause: new Error('cleanup failed') })
+ const cleanupError = new SourceCleanupError(space.realPath, trashFile, { cause: new Error('cleanup failed') })
prepareFileTransfer(space.realPath, trashFile)
- vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)
+ filesTasksTransfer.delete.mockResolvedValueOnce(cleanupError)
await expect(service.delete(user, space, undefined, signal)).rejects.toBe(cleanupError)
- expect(taskUtils.moveAbortable).toHaveBeenCalledWith(
- space.realPath,
- trashFile,
- expect.objectContaining({
- cacheKey: 'task-delete',
- onTransferStart: expect.any(Function),
- overwrite: true,
- signal,
- stagingDir: user.tasksPath
- })
- )
+ expect(filesTasksTransfer.delete).toHaveBeenCalledWith(user, space, trashFile, false, signal, expect.any(Function))
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, false)
})
@@ -1023,7 +1011,7 @@ describe(FilesManager.name, () => {
await service.delete(user, space)
expect(filesUtils.moveFiles).toHaveBeenCalledWith(space.realPath, trashFile, true)
- expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ expect(filesTasksTransfer.delete).not.toHaveBeenCalled()
})
})
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index be7ec99c..8997308e 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -29,7 +29,7 @@ import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
import { FileDBProps } from '../interfaces/file-db-props.interface'
import { FileLock } from '../interfaces/file-lock.interface'
import { FileLockProps } from '../interfaces/file-props.interface'
-import { FileError } from '../models/file-error'
+import { FileError, SourceCleanupError } from '../models/file-error'
import { LockConflict } from '../models/file-lock-error'
import {
checkFileName,
@@ -66,7 +66,8 @@ import { ACTION } from '../../../common/constants'
import { pipeline } from 'node:stream/promises'
import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'
import { FILE_ERROR_MESSAGES, maxFileSizeExceededError } from '../utils/errors'
-import { copyAbortable, createTaskTemporaryDir, moveAbortable, SourceCleanupError, taskTemporaryPath } from '../utils/tasks'
+import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'
+import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'
@Injectable()
export class FilesManager {
@@ -79,7 +80,8 @@ export class FilesManager {
private readonly spacesManager: SpacesManager,
private readonly contextManager: ContextManager,
private readonly notificationsManager: NotificationsManager,
- public readonly filesLockManager: FilesLockManager
+ public readonly filesLockManager: FilesLockManager,
+ private readonly filesTasksTransfer: FilesTasksTransfer
) {}
sendFileFromSpace(space: SpaceEnv, downloadName = ''): SendFile {
@@ -387,6 +389,7 @@ export class FilesManager {
signal?: AbortSignal
): Promise<void> {
const isTaskContext = Boolean(srcSpace.task?.cacheKey)
+ const useTaskTransfer = isTaskContext && (!isMove || Boolean(signal))
// checks
this.checkNotTrashRepository(dstSpace)
if (!canAccessToSpace(user, dstSpace)) {
@@ -476,15 +479,15 @@ export class FilesManager {
await this.filesLockManager.checkConflicts(dstSpace.dbFile, depth, { userId: user.id, lockTokens: dav?.lockTokens })
// Task transfers defer overwrite handling until their staged content is ready to commit.
- if (!isTaskContext && overwrite && (await isPathExists(dstSpace.realPath))) {
+ if (!useTaskTransfer && overwrite && (await isPathExists(dstSpace.realPath))) {
await this.delete(user, dstSpace)
}
// do
if (isMove) {
let sourceCleanupError: SourceCleanupError | undefined
- if (isTaskContext) {
- sourceCleanupError = await this.moveAsTask(user, srcSpace, dstSpace, overwrite, isDir, signal)
+ if (useTaskTransfer) {
+ sourceCleanupError = await this.filesTasksTransfer.move(user, srcSpace, dstSpace, overwrite, isDir, signal, () => this.delete(user, dstSpace))
} else {
await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
}
@@ -504,7 +507,7 @@ export class FilesManager {
if (!signal) {
throw new Error('An abort signal is required for a copy task')
}
- await this.copyAsTask(user, srcSpace, dstSpace, overwrite, recursive, isDir, signal)
+ await this.filesTasksTransfer.copy(user, srcSpace, dstSpace, overwrite, recursive, isDir, signal, () => this.delete(user, dstSpace))
} else {
await copyFiles(srcSpace.realPath, dstSpace.realPath, overwrite, recursive)
}
@@ -540,7 +543,9 @@ export class FilesManager {
await makeDir(trashDir, true)
}
if (isTaskContext) {
- sourceCleanupError = await this.deleteAsTask(user, space, trashFile, isDir, signal)
+ sourceCleanupError = await this.filesTasksTransfer.delete(user, space, trashFile, isDir, signal, () =>
+ this.moveExistingTrashFile(space, trashFile)
+ )
} else {
await this.moveExistingTrashFile(space, trashFile)
await moveFiles(space.realPath, trashFile, true)
@@ -821,92 +826,6 @@ export class FilesManager {
}
}
- private async copyAsTask(
- user: UserModel,
- srcSpace: SpaceEnv,
- dstSpace: SpaceEnv,
- overwrite: boolean,
- recursive: boolean,
- isDir: boolean,
- signal: AbortSignal
- ): Promise<void> {
- await this.setTaskTotalSize(srcSpace, isDir)
- await copyAbortable(srcSpace.realPath, dstSpace.realPath, {
- beforeCommit: this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite),
- cacheKey: srcSpace.task!.cacheKey,
- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),
- overwrite,
- recursive,
- signal,
- stagingDir: user.tasksPath
- })
- }
-
- private async moveAsTask(
- user: UserModel,
- srcSpace: SpaceEnv,
- dstSpace: SpaceEnv,
- overwrite: boolean,
- isDir: boolean,
- signal?: AbortSignal
- ): Promise<SourceCleanupError | undefined> {
- await this.setTaskTotalSize(srcSpace, isDir)
- const beforeCommit = this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite)
- if (!signal) {
- // Same-device moves stay on the regular atomic path and are therefore not cancellable.
- this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)
- await beforeCommit?.()
- await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
- return
- }
- return moveAbortable(srcSpace.realPath, dstSpace.realPath, {
- beforeCommit,
- cacheKey: srcSpace.task!.cacheKey,
- crossDevice: true,
- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),
- overwrite,
- signal,
- stagingDir: user.tasksPath
- })
- }
-
- private async deleteAsTask(
- user: UserModel,
- space: SpaceEnv,
- trashFile: string,
- isDir: boolean,
- signal?: AbortSignal
- ): Promise<SourceCleanupError | undefined> {
- await this.setTaskTotalSize(space, isDir)
- const beforeCommit = () => this.moveExistingTrashFile(space, trashFile)
- if (!signal) {
- // Moving to trash on the same device stays atomic; only cross-device deletes receive a signal.
- this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)
- await beforeCommit()
- await moveFiles(space.realPath, trashFile, true)
- return
- }
- return moveAbortable(space.realPath, trashFile, {
- beforeCommit,
- cacheKey: space.task!.cacheKey,
- crossDevice: true,
- onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),
- overwrite: true,
- signal,
- stagingDir: user.tasksPath
- })
- }
-
- private prepareTaskDestination(user: UserModel, srcSpace: SpaceEnv, dstSpace: SpaceEnv, overwrite: boolean): (() => Promise<void>) | undefined {
- if (!overwrite || srcSpace.realPath.toLowerCase() === dstSpace.realPath.toLowerCase()) return
- // Preserve the current destination until the staged transfer reaches its commit phase.
- return async () => {
- if (await isPathExists(dstSpace.realPath)) {
- await this.delete(user, dstSpace)
- }
- }
- }
-
private async moveExistingTrashFile(space: SpaceEnv, trashFile: string): Promise<void> {
if (!(await isPathExists(trashFile))) return
const dstTrash = await uniqueDatedFilePath(trashFile)
@@ -916,16 +835,6 @@ export class FilesManager {
await this.filesQueries.moveFiles(trashFileDB, dstTrashFileDB, dstTrash.isDir)
}
- private async setTaskTotalSize(space: SpaceEnv, isDir: boolean): Promise<void> {
- if (!isDir) {
- space.task!.props.totalSize = await fileSize(space.realPath)
- }
- }
-
- private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {
- FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)
- }
-
private logSourceCleanupError(error: SourceCleanupError): void {
this.logger.error({
tag: this.logSourceCleanupError.name,
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 9924f45a..64873beb 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -19,7 +19,7 @@ import { files } from '../schemas/files.schema'
import { isPathExists, removeFiles } from '../utils/files'
import { taskTemporaryPrefix } from '../utils/tasks'
import { FilesContentIndexer } from './files-content-indexer.service'
-import { FilesTasksManager } from './files-tasks-manager.service'
+import { FilesTasksManager } from './tasks/files-tasks-manager.service'
import { FilesQuotaManager } from './files-quota-manager.service'
import { FilesTrashRetention } from './files-trash-retention.service'
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
similarity index 84%
rename from backend/src/applications/files/services/files-tasks-manager.service.spec.ts
rename to backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
index 9b9c9037..becac1ba 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
@@ -1,21 +1,24 @@
import { Test, TestingModule } from '@nestjs/testing'
import { HttpException, HttpStatus } from '@nestjs/common'
import crypto from 'node:crypto'
-import { Cache } from '../../../infrastructure/cache/cache.service'
-import { SpacesManager } from '../../spaces/services/spaces-manager.service'
-import * as filesUtils from '../utils/files'
-import { FileTaskEvent } from '../events/file-events'
-import { SendFile } from '../utils/send-file'
-import { FILE_OPERATION } from '../constants/operations'
-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
-import { FileTaskStatus } from '../models/file-task'
-import { FilesMethods } from './files-methods.service'
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { SpacesManager } from '../../../spaces/services/spaces-manager.service'
+import * as filesUtils from '../../utils/files'
+import * as tasksUtils from '../../utils/tasks'
+import { FileTaskEvent } from '../../events/file-events'
+import { SendFile } from '../../utils/send-file'
+import { FILE_OPERATION } from '../../constants/operations'
+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'
+import { FileTaskStatus } from '../../models/file-task'
+import { FilesMethods } from '../files-methods.service'
import { FilesTasksManager } from './files-tasks-manager.service'
import { FilesTasksQueue } from './files-tasks-queue.service'
+import { FilesTasksWatcher } from './files-tasks-watcher.service'
import { Mock } from 'vitest'
describe(FilesTasksManager.name, () => {
let filesTasksManager: FilesTasksManager
+ let filesTasksWatcher: FilesTasksWatcher
let module: TestingModule
let filesMethods: {
doWork: Mock
@@ -90,6 +93,7 @@ describe(FilesTasksManager.name, () => {
providers: [
FilesTasksManager,
FilesTasksQueue,
+ FilesTasksWatcher,
{
provide: Cache,
useValue: cache
@@ -101,6 +105,7 @@ describe(FilesTasksManager.name, () => {
module.useLogger(['fatal'])
filesTasksManager = module.get<FilesTasksManager>(FilesTasksManager)
+ filesTasksWatcher = module.get<FilesTasksWatcher>(FilesTasksWatcher)
})
afterEach(async () => {
@@ -118,14 +123,14 @@ describe(FilesTasksManager.name, () => {
const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')
const space = { task: { cacheKey, props: {} } } as any
cacheStore.set(cacheKey, { props: {} })
- ;(filesTasksManager as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')
+ ;(filesTasksWatcher as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')
- const watcher = (filesTasksManager as any).tasksWatcher[cacheKey] as NodeJS.Timeout
+ const watcher = (filesTasksWatcher as any).tasksWatcher[cacheKey] as NodeJS.Timeout
expect(watcher.hasRef()).toBe(false)
- filesTasksManager.onModuleDestroy()
+ filesTasksWatcher.onModuleDestroy()
- expect((filesTasksManager as any).tasksWatcher).toEqual({})
+ expect((filesTasksWatcher as any).tasksWatcher).toEqual({})
expect(FileTaskEvent.listenerCount('startWatch')).toBe(listenersBeforeDestroy - 1)
})
@@ -139,7 +144,7 @@ describe(FilesTasksManager.name, () => {
finishUpdate = resolve
})
)
- ;(filesTasksManager as any).watchTask('task-1', update)
+ ;(filesTasksWatcher as any).watchTask('task-1', update)
await vi.advanceTimersByTimeAsync(2000)
expect(update).toHaveBeenCalledTimes(1)
@@ -148,7 +153,7 @@ describe(FilesTasksManager.name, () => {
await Promise.resolve()
await vi.advanceTimersByTimeAsync(1000)
expect(update).toHaveBeenCalledTimes(2)
- ;(filesTasksManager as any).stopWatch('task-1')
+ ;(filesTasksWatcher as any).stopWatch('task-1')
} finally {
vi.useRealTimers()
}
@@ -214,7 +219,7 @@ describe(FilesTasksManager.name, () => {
})
it('should watch a temporary path while keeping the published file name', () => {
- const startWatchSpy = vi.spyOn(filesTasksManager as any, 'startWatch').mockResolvedValueOnce(undefined)
+ const startWatchSpy = vi.spyOn(filesTasksWatcher as any, 'startWatch').mockResolvedValueOnce(undefined)
const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any
FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')
@@ -232,10 +237,10 @@ describe(FilesTasksManager.name, () => {
it('should watch the published archive after the temporary archive has been moved', async () => {
const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)
- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
const space = { task: { cacheKey: 'task-1', props: {} } } as any
- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
@@ -248,10 +253,10 @@ describe(FilesTasksManager.name, () => {
const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })
const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)
const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)
- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
const space = { task: { cacheKey: 'task-1', props: {} } } as any
- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
@@ -264,32 +269,32 @@ describe(FilesTasksManager.name, () => {
it('should retry the published directory when the temporary directory is moved while reading it', async () => {
const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)
const countDirEntriesSpy = vi
- .spyOn(filesUtils, 'countDirEntries')
- .mockResolvedValueOnce({ files: 0, directories: 0 })
- .mockResolvedValueOnce({ files: 2, directories: 1 })
- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ .spyOn(tasksUtils, 'countDirEntriesAndSize')
+ .mockResolvedValueOnce({ files: 0, directories: 0, size: 0 })
+ .mockResolvedValueOnce({ files: 2, directories: 1, size: 42 })
+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
const space = { task: { cacheKey: 'task-1', props: {} } } as any
- await (filesTasksManager as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')
+ await (filesTasksWatcher as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')
expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')
expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')
expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')
- expect(space.task.props).toEqual({ files: 2, directories: 1 })
- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1 })
+ expect(space.task.props).toEqual({ files: 2, directories: 1, size: 42 })
+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1, size: 42 })
})
it('should silently skip watcher update while temporary and published paths are missing', async () => {
vi.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)
const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize')
- const loggerErrorSpy = vi.spyOn((filesTasksManager as any).logger, 'error')
- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)
- const updateTaskSpy = vi.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')
+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)
+ const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
const space = { task: { cacheKey: 'task-1', props: {} } } as any
- await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+ await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
expect(loggerErrorSpy).not.toHaveBeenCalled()
expect(stopWatchSpy).not.toHaveBeenCalled()
@@ -300,11 +305,11 @@ describe(FilesTasksManager.name, () => {
it('should stop a watcher without staging when its path is missing', async () => {
const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })
vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)
- const loggerErrorSpy = vi.spyOn((filesTasksManager as any).logger, 'error')
- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)
+ const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')
+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)
const space = { task: { cacheKey: 'task-1', props: {} } } as any
- await (filesTasksManager as any).updateCompressTask(space, '/files/archive.tgz')
+ await (filesTasksWatcher as any).updateCompressTask(space, '/files/archive.tgz')
expect(loggerErrorSpy).toHaveBeenCalled()
expect(stopWatchSpy).toHaveBeenCalledWith('task-1')
@@ -331,7 +336,7 @@ describe(FilesTasksManager.name, () => {
return cacheStore.get(key)
})
- const updatePromise = (filesTasksManager as any).updateTask(cacheKey, { size: 20 })
+ const updatePromise = (filesTasksWatcher as any).updateTask(cacheKey, { size: 20 })
await getStarted
const completionPromise = (filesTasksManager as any).setTaskDone(cacheKey, FileTaskStatus.SUCCESS, 'done')
releaseGet()
@@ -367,6 +372,67 @@ describe(FilesTasksManager.name, () => {
expect(storedTask.endedAt).toBeDefined()
})
+ it('should recalculate final copy properties from the published path', async () => {
+ const taskId = '77777777-7777-4777-8777-777777777777'
+ const user = { id: 7 } as any
+ const space = { url: 'files/personal/source' } as any
+ const result = { path: 'files/personal', name: 'destination', mime: 'directory' }
+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)
+ filesMethods.doWork.mockResolvedValueOnce(result)
+ const getPathPropsSpy = vi.spyOn(filesTasksWatcher, 'getPathProps').mockResolvedValueOnce({ directories: 2, files: 3, size: 120 })
+
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, space, null, 'doWork')
+ await flushPromises()
+
+ expect(spacesManager.spaceEnv).toHaveBeenCalledWith(user, ['files', 'personal', 'destination'])
+ expect(getPathPropsSpy).toHaveBeenCalledWith('/data/users/john/files/destination.txt')
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({
+ status: FileTaskStatus.SUCCESS,
+ path: result.path,
+ name: result.name,
+ props: {
+ directories: 2,
+ files: 3,
+ progress: 100,
+ size: 120
+ }
+ })
+ })
+
+ it('should finalize a fast delete task without waiting for a watcher tick', async () => {
+ const taskId = '66666666-6666-4666-8666-666666666666'
+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)
+ filesMethods.doWork.mockImplementationOnce(async (_user, taskSpace) => {
+ taskSpace.task.props = {
+ directories: 2,
+ files: 3,
+ progress: 1,
+ size: 0,
+ totalSize: 120
+ }
+ })
+ const user = { id: 7 } as any
+ const space = {
+ inTrashRepository: true,
+ realPath: '/data/users/john/trash/document',
+ url: 'trash/personal/document'
+ } as any
+
+ await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')
+ await flushPromises()
+
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({
+ status: FileTaskStatus.SUCCESS,
+ props: {
+ directories: 2,
+ files: 3,
+ progress: 100,
+ size: 120,
+ totalSize: 120
+ }
+ })
+ })
+
it('should create a task and mark it as error when the async method fails', async () => {
vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
filesMethods.doWork.mockRejectedValueOnce(new Error('operation failed'))
@@ -538,7 +604,7 @@ describe(FilesTasksManager.name, () => {
it('should delete completed task, remove archive file and delete cache entry', async () => {
const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValueOnce(undefined)
- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)
+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)
const user = { id: 22, tasksPath: '/tmp/tasks' } as any
cacheStore.set(`${CACHE_TASK_PREFIX}-22-task-ok`, {
id: 'task-ok',
@@ -556,7 +622,7 @@ describe(FilesTasksManager.name, () => {
it('should ignore active tasks when deleting all tasks', async () => {
const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)
- const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValue(undefined)
+ const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValue(undefined)
const user = { id: 31, tasksPath: '/tmp/tasks' } as any
cacheStore.set(`${CACHE_TASK_PREFIX}-31-task-pending`, {
id: 'task-pending',
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
similarity index 56%
rename from backend/src/applications/files/services/files-tasks-manager.service.ts
rename to backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
index 3b71c47e..5d096321 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
@@ -2,45 +2,40 @@ import { HttpException, HttpStatus, Injectable, Logger, OnModuleDestroy, Streama
import { FastifyReply } from 'fastify'
import crypto from 'node:crypto'
import path from 'node:path'
-import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'
-import { currentTimeStamp } from '../../../common/shared'
-import { Cache } from '../../../infrastructure/cache/cache.service'
-import { SpaceEnv } from '../../spaces/models/space-env.model'
-import { SpacesManager } from '../../spaces/services/spaces-manager.service'
-import { realTrashPathFromSpace } from '../../spaces/utils/paths'
-import { UserModel } from '../../users/models/user.model'
-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
-import { FILE_OPERATION } from '../constants/operations'
-import { CopyMoveFileDto } from '../dto/file-operations.dto'
-import { FileTaskEvent } from '../events/file-events'
-import type { FileTaskQueueItem } from '../interfaces/file-task-queue.interface'
-import type { FileTasksPollResponse } from '../interfaces/file-task.interface'
-import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'
-import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'
-import { SendFile } from '../utils/send-file'
-import { isTaskCancellable } from '../utils/tasks'
-import { FilesMethods } from './files-methods.service'
+import { FastifyAuthenticatedRequest } from '../../../../authentication/interfaces/auth-request.interface'
+import { currentTimeStamp } from '../../../../common/shared'
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { SpaceEnv } from '../../../spaces/models/space-env.model'
+import { SpacesManager } from '../../../spaces/services/spaces-manager.service'
+import { realTrashPathFromSpace } from '../../../spaces/utils/paths'
+import { UserModel } from '../../../users/models/user.model'
+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../../constants/cache'
+import { FILE_OPERATION } from '../../constants/operations'
+import { CopyMoveFileDto } from '../../dto/file-operations.dto'
+import type { FileTaskQueueItem } from '../../interfaces/file-task-queue.interface'
+import type { FileTasksPollResponse } from '../../interfaces/file-task.interface'
+import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'
+import { dirName, fileName, removeFiles } from '../../utils/files'
+import { SendFile } from '../../utils/send-file'
+import { isTaskCancellable } from '../../utils/tasks'
+import { FilesMethods } from '../files-methods.service'
import { FilesTasksQueue } from './files-tasks-queue.service'
+import { FilesTasksWatcher } from './files-tasks-watcher.service'
@Injectable()
export class FilesTasksManager implements OnModuleDestroy {
// Task cache key: `ftask-${userId}-${taskId}`.
private readonly logger = new Logger(FilesTasksManager.name)
private readonly watchInterval = 1000
- private tasksWatcher: Record<string, NodeJS.Timeout> = {}
private tasksCancellationWatcher: Record<string, NodeJS.Timeout> = {}
- // Cache writes are serialized per task, while finalization blocks any late progress write.
- private pendingTaskUpdates = new Map<string, Promise<void>>()
- private finalizingTasks = new Set<string>()
constructor(
private readonly cache: Cache,
private readonly filesMethods: FilesMethods,
private readonly filesTasksQueue: FilesTasksQueue,
+ private readonly filesTasksWatcher: FilesTasksWatcher,
private readonly spacesManager: SpacesManager
- ) {
- FileTaskEvent.on('startWatch', this.onStartWatch)
- }
+ ) {}
static getCacheKey(userId: number, taskId?: string): string {
return `${CACHE_TASK_PREFIX}-${userId}-${taskId || '*'}`
@@ -51,10 +46,6 @@ export class FilesTasksManager implements OnModuleDestroy {
}
onModuleDestroy(): void {
- FileTaskEvent.off('startWatch', this.onStartWatch)
- for (const cacheKey of Object.keys(this.tasksWatcher)) {
- this.stopWatch(cacheKey)
- }
for (const cacheKey of Object.keys(this.tasksCancellationWatcher)) {
this.stopCancellationWatch(cacheKey)
}
@@ -115,7 +106,7 @@ export class FilesTasksManager implements OnModuleDestroy {
removeFiles(rPath).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))
}
// clear watcher
- this.stopWatch(key)
+ this.filesTasksWatcher.stopWatch(key)
// remove from cache
this.cache.del(key).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))
}
@@ -167,12 +158,10 @@ export class FilesTasksManager implements OnModuleDestroy {
}
}
- private async setTaskDone(cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {
- this.finalizingTasks.add(cacheKey)
+ private async setTaskDone(cacheKey: string, status: FileTaskStatus, result: any, finalProps?: FileTaskProps): Promise<void> {
+ await this.filesTasksWatcher.beginFinalization(cacheKey)
try {
- this.stopWatch(cacheKey)
this.stopCancellationWatch(cacheKey)
- await this.pendingTaskUpdates.get(cacheKey)?.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
const task: FileTask = await this.cache.get(cacheKey)
if (task) {
task.status = status
@@ -184,13 +173,16 @@ export class FilesTasksManager implements OnModuleDestroy {
Object.assign(task, result)
}
}
+ if (finalProps) {
+ task.props = { ...task.props, ...finalProps }
+ }
await this.cache.set(cacheKey, task, CACHE_TASK_TTL)
}
await this.cache
.del(cacheKey.replace(`${CACHE_TASK_PREFIX}-`, `${CACHE_TASK_CANCEL_PREFIX}-`))
.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
} finally {
- this.finalizingTasks.delete(cacheKey)
+ this.filesTasksWatcher.endFinalization(cacheKey)
}
}
@@ -212,9 +204,10 @@ export class FilesTasksManager implements OnModuleDestroy {
? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)
: this.filesMethods[task.method](task.user, task.space, task.dto)
taskPromise
- .then((data: any) => {
+ .then(async (data: any) => {
this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })
- this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) =>
+ const finalProps = await this.getFinalTransferProps(task, data)
+ this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data, finalProps).catch((e: Error) =>
this.logger.warn({ tag: this.runTask.name, msg: `${e}` })
)
})
@@ -230,14 +223,35 @@ export class FilesTasksManager implements OnModuleDestroy {
})
}
- private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {
+ private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any, finalProps?: FileTaskProps): Promise<void> {
try {
- await this.setTaskDone(cacheKey, status, result)
+ await this.setTaskDone(cacheKey, status, result, finalProps)
} finally {
await this.filesTasksQueue.releaseAndDrain(userId)
}
}
+ private async getFinalTransferProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {
+ if (task.task.type === FILE_OPERATION.DELETE) {
+ const props = task.space.task?.props
+ return {
+ ...props,
+ ...(props?.totalSize !== undefined ? { size: props.totalSize } : {}),
+ progress: 100
+ }
+ }
+ if (task.task.type !== FILE_OPERATION.COPY && task.task.type !== FILE_OPERATION.MOVE) return
+ if (!result?.path || !result?.name) return
+ try {
+ const publishedUrl = path.join(result.path, result.name)
+ const publishedSpace = await this.spacesManager.spaceEnv(task.user, publishedUrl.split('/'))
+ const props = await this.filesTasksWatcher.getPathProps(publishedSpace.realPath)
+ return { ...props, progress: 100 }
+ } catch (e) {
+ this.logger.warn({ tag: this.getFinalTransferProps.name, msg: `${e}` })
+ }
+ }
+
private isActiveStatus(status: FileTaskStatus): boolean {
return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
}
@@ -265,96 +279,6 @@ export class FilesTasksManager implements OnModuleDestroy {
}
}
- private updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {
- if (this.finalizingTasks.has(cacheKey)) return Promise.resolve()
- const previousUpdate = this.pendingTaskUpdates.get(cacheKey) ?? Promise.resolve()
- const update = previousUpdate
- .catch(() => undefined)
- .then(async () => {
- if (this.finalizingTasks.has(cacheKey)) return
- let ftask: FileTask = await this.cache.get(cacheKey)
- if (!ftask) {
- this.stopWatch(cacheKey)
- return
- }
- // Finalization may have started while the cache read was pending.
- if (!this.isActiveStatus(ftask.status) || this.finalizingTasks.has(cacheKey)) {
- this.stopWatch(cacheKey)
- return
- }
- if (task) ftask = { ...ftask, ...task }
- if (props) ftask.props = { ...ftask.props, ...props }
- await this.cache.set(cacheKey, ftask, CACHE_TASK_TTL)
- })
- this.pendingTaskUpdates.set(cacheKey, update)
- return update.finally(() => {
- if (this.pendingTaskUpdates.get(cacheKey) === update) this.pendingTaskUpdates.delete(cacheKey)
- })
- }
-
- private startWatch(
- space: SpaceEnv,
- taskType: FILE_OPERATION,
- rPath: string,
- taskPath?: string,
- taskName = fileName(rPath),
- publishedPath?: string
- ): void {
- const taskContext = space.task
- if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return
- const { cacheKey, props } = taskContext
- this.logger.verbose({ tag: this.startWatch.name, msg: cacheKey })
- this.updateTask(cacheKey, props, {
- name: taskName,
- path: taskPath
- }).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
- switch (taskType) {
- case FILE_OPERATION.COMPRESS:
- this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.DECOMPRESS:
- this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.DOWNLOAD:
- this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.COPY:
- case FILE_OPERATION.DELETE:
- case FILE_OPERATION.MOVE:
- this.watchTask(cacheKey, () => this.updateCopyMoveTask(space, rPath))
- return
- default:
- this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })
- return
- }
- }
-
- private stopWatch(cacheKey: string): void {
- if (!(cacheKey in this.tasksWatcher)) return
- clearInterval(this.tasksWatcher[cacheKey])
- delete this.tasksWatcher[cacheKey]
- }
-
- private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {
- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)
- }
-
- private watchTask(cacheKey: string, update: () => Promise<void>): void {
- // Directory scans may exceed the interval, so ticks use exhaust semantics.
- let updateInProgress = false
- const watcher = setInterval(() => {
- if (updateInProgress) return
- updateInProgress = true
- void update()
- .catch((e: Error) => this.logger.error({ tag: this.watchTask.name, msg: `${e}` }))
- .finally(() => {
- updateInProgress = false
- })
- }, this.watchInterval)
- watcher.unref()
- this.tasksWatcher[cacheKey] = watcher
- }
-
private watchCancellation(task: FileTask, userId: number, cacheKey: string): AbortController | undefined {
if (!task.cancellable) return
const controller = new AbortController()
@@ -382,77 +306,4 @@ export class FilesTasksManager implements OnModuleDestroy {
clearInterval(this.tasksCancellationWatcher[cacheKey])
delete this.tasksCancellationWatcher[cacheKey]
}
-
- private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
- if (size === undefined) return
- space.task.props.size = size
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntries)
- if (props === undefined) return
- space.task.props = props
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async updateCopyMoveTask(space: SpaceEnv, rPath: string): Promise<void> {
- try {
- if (await isPathIsDir(rPath)) {
- space.task.props = await countDirEntries(rPath)
- } else {
- await this.calcSizeAndProgressTask(space, rPath)
- }
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateCopyMoveTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {
- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
- if (size === undefined) return false
- space.task.props.size = size
- if (space.task.props.totalSize) {
- space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize
- }
- return true
- }
-
- private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {
- if (!publishedPath) return read(rPath)
- if (await isPathExists(rPath)) {
- try {
- const value = await read(rPath)
- // The temporary path may be moved atomically while it is being read.
- if (await isPathExists(rPath)) return value
- } catch (e) {
- // Retry the published path only when the temporary path disappeared during publication.
- if (e?.code !== 'ENOENT') throw e
- }
- }
- if (await isPathExists(publishedPath)) return read(publishedPath)
- }
}
diff --git a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.ts
similarity index 96%
rename from backend/src/applications/files/services/files-tasks-queue.service.spec.ts
rename to backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.ts
index 3f45541d..28e67491 100644
--- a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-queue.service.spec.ts
@@ -1,7 +1,7 @@
import { Mock } from 'vitest'
-import { Cache } from '../../../infrastructure/cache/cache.service'
-import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
-import type { FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'
+import type { FileTaskQueueItem, FileTaskQueueStarter } from '../../interfaces/file-task-queue.interface'
import { FilesTasksQueue } from './files-tasks-queue.service'
describe(FilesTasksQueue.name, () => {
diff --git a/backend/src/applications/files/services/files-tasks-queue.service.ts b/backend/src/applications/files/services/tasks/files-tasks-queue.service.ts
similarity index 94%
rename from backend/src/applications/files/services/files-tasks-queue.service.ts
rename to backend/src/applications/files/services/tasks/files-tasks-queue.service.ts
index 75f02f87..c7216c94 100644
--- a/backend/src/applications/files/services/files-tasks-queue.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-queue.service.ts
@@ -1,7 +1,7 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'
-import { Cache } from '../../../infrastructure/cache/cache.service'
-import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
-import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../../constants/cache'
+import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../../interfaces/file-task-queue.interface'
@Injectable()
export class FilesTasksQueue implements OnModuleDestroy {
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
new file mode 100644
index 00000000..30884f9d
--- /dev/null
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
@@ -0,0 +1,193 @@
+import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import * as filesUtils from '../../utils/files'
+import { FilesTasksTransfer } from './files-tasks-transfer.service'
+import { SourceCleanupError } from '../../models/file-error'
+
+describe(FilesTasksTransfer.name, () => {
+ const cacheKey = 'ftask-7-task-id'
+ let service: FilesTasksTransfer
+ let tmpDir: string
+ let srcPath: string
+ let dstPath: string
+ let stagingDir: string
+
+ beforeEach(async () => {
+ service = new FilesTasksTransfer()
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))
+ srcPath = path.join(tmpDir, 'source.txt')
+ dstPath = path.join(tmpDir, 'destination.txt')
+ stagingDir = path.join(tmpDir, 'tasks')
+ })
+
+ afterEach(async () => {
+ await rm(tmpDir, { recursive: true, force: true })
+ vi.restoreAllMocks()
+ })
+
+ const copyAbortable = (options: Record<string, any>) => (service as any).copyAbortable(srcPath, dstPath, options)
+ const moveAbortable = (options: Record<string, any>) => (service as any).moveAbortable(srcPath, dstPath, options)
+
+ const mockCrossDevice = () => {
+ vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)
+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
+ }
+
+ it('updates task size and progress through its public copy operation', async () => {
+ const content = 'content'
+ const srcSpace = {
+ realPath: srcPath,
+ task: { cacheKey, props: {} }
+ } as any
+ const dstSpace = { realPath: dstPath } as any
+ await writeFile(srcPath, content)
+
+ await service.copy({ tasksPath: stagingDir } as any, srcSpace, dstSpace, false, false, false, new AbortController().signal, vi.fn())
+
+ expect(srcSpace.task.props).toMatchObject({
+ progress: 100,
+ size: Buffer.byteLength(content),
+ totalSize: Buffer.byteLength(content)
+ })
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)
+ })
+
+ it('publishes a copied file only after the commit hook', async () => {
+ const signal = new AbortController().signal
+ let temporaryPath = ''
+ let transferredBytes = 0
+ await writeFile(srcPath, 'content')
+
+ await copyAbortable({
+ beforeCommit: async () => {
+ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')
+ },
+ cacheKey,
+ onTransferStart: (value: string) => {
+ temporaryPath = value
+ },
+ onProgress: (bytes: number) => {
+ transferredBytes += bytes
+ },
+ signal,
+ stagingDir
+ })
+
+ expect(transferredBytes).toBe(Buffer.byteLength('content'))
+ expect(path.dirname(temporaryPath)).toBe(stagingDir)
+ expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ })
+
+ it('cleans the temporary destination when copy is cancelled', async () => {
+ const controller = new AbortController()
+ await writeFile(srcPath, 'content')
+ controller.abort()
+
+ await expect(copyAbortable({ cacheKey, signal: controller.signal, stagingDir })).rejects.toMatchObject({ name: 'AbortError' })
+
+ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('copies directly to a destination on another device', async () => {
+ const signal = new AbortController().signal
+ let watchedPath = ''
+ let destinationPrepared = false
+ await writeFile(srcPath, 'content')
+ mockCrossDevice()
+
+ await copyAbortable({
+ beforeCommit: async () => {
+ destinationPrepared = true
+ },
+ cacheKey,
+ onTransferStart: (value: string) => {
+ expect(destinationPrepared).toBe(true)
+ watchedPath = value
+ },
+ signal,
+ stagingDir
+ })
+
+ expect(watchedPath).toBe(dstPath)
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('keeps an existing direct-copy destination when overwrite is rejected', async () => {
+ const signal = new AbortController().signal
+ await writeFile(srcPath, 'new content')
+ await writeFile(dstPath, 'existing content')
+ mockCrossDevice()
+
+ await expect(copyAbortable({ cacheKey, signal, stagingDir })).rejects.toMatchObject({ code: 'EEXIST' })
+
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
+ })
+
+ it('does not prepare a direct-copy destination when already cancelled', async () => {
+ const controller = new AbortController()
+ const beforeCommit = vi.fn()
+ await writeFile(srcPath, 'new content')
+ await writeFile(dstPath, 'existing content')
+ await fs.mkdir(stagingDir)
+ controller.abort()
+ mockCrossDevice()
+
+ await expect(copyAbortable({ beforeCommit, cacheKey, overwrite: true, signal: controller.signal, stagingDir })).rejects.toMatchObject({
+ name: 'AbortError'
+ })
+
+ expect(beforeCommit).not.toHaveBeenCalled()
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
+ })
+
+ it('preserves a direct-copy error when destination cleanup also fails', async () => {
+ const signal = new AbortController().signal
+ const transferError = Object.assign(new Error('transfer failed'), { code: 'EIO' })
+ await fs.mkdir(stagingDir)
+ vi.spyOn(fs, 'lstat').mockImplementation(async (rPath) => {
+ if (rPath === stagingDir) return { dev: 1 } as any
+ throw transferError
+ })
+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(new Error('cleanup failed'))
+
+ await expect(copyAbortable({ cacheKey, signal, stagingDir })).rejects.toBe(transferError)
+ })
+
+ it('copies then removes the source for a cross-device move', async () => {
+ const signal = new AbortController().signal
+ await writeFile(srcPath, 'content')
+
+ await moveAbortable({ cacheKey, crossDevice: true, signal, stagingDir })
+
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('reports source cleanup failure after keeping the published destination', async () => {
+ const signal = new AbortController().signal
+ const error = new Error('source cleanup failed')
+ await writeFile(srcPath, 'content')
+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)
+
+ const cleanupError = await moveAbortable({ cacheKey, crossDevice: true, signal, stagingDir })
+
+ expect(cleanupError).toBeInstanceOf(SourceCleanupError)
+ expect(cleanupError).toMatchObject({
+ cause: error,
+ dstPath,
+ message: 'Destination was published but the source could not be removed',
+ name: SourceCleanupError.name,
+ srcPath
+ })
+ await expect(readFile(srcPath, 'utf8')).resolves.toBe('content')
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ })
+})
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
new file mode 100644
index 00000000..092f299a
--- /dev/null
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
@@ -0,0 +1,259 @@
+import { Injectable } from '@nestjs/common'
+import { createReadStream, createWriteStream } from 'node:fs'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { Transform } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { UserModel } from '../../../users/models/user.model'
+import { SpaceEnv } from '../../../spaces/models/space-env.model'
+import { DEFAULT_HIGH_WATER_MARK } from '../../constants/files'
+import { FILE_OPERATION } from '../../constants/operations'
+import { FileTaskEvent } from '../../events/file-events'
+import { fileSize, isPathExists, moveFiles, removeFiles } from '../../utils/files'
+import { countDirEntriesAndSize, isCrossDeviceMove, taskTemporaryPath } from '../../utils/tasks'
+import { FileTaskCopyTaskOptions, FileTaskTransferOptions } from '../../interfaces/file-task.interface'
+import { SourceCleanupError } from '../../models/file-error'
+
+@Injectable()
+export class FilesTasksTransfer {
+ async copy(
+ user: UserModel,
+ srcSpace: SpaceEnv,
+ dstSpace: SpaceEnv,
+ overwrite: boolean,
+ recursive: boolean,
+ isDir: boolean,
+ signal: AbortSignal,
+ deleteDestination: () => Promise<void>
+ ): Promise<void> {
+ await this.initializeTaskProps(srcSpace, isDir)
+ await this.copyAbortable(srcSpace.realPath, dstSpace.realPath, {
+ beforeCommit: this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination),
+ cacheKey: srcSpace.task!.cacheKey,
+ onProgress: (bytes) => this.updateProgress(srcSpace, bytes),
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),
+ overwrite,
+ recursive,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ async move(
+ user: UserModel,
+ srcSpace: SpaceEnv,
+ dstSpace: SpaceEnv,
+ overwrite: boolean,
+ isDir: boolean,
+ signal: AbortSignal | undefined,
+ deleteDestination: () => Promise<void>
+ ): Promise<SourceCleanupError | undefined> {
+ await this.initializeTaskProps(srcSpace, isDir)
+ const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)
+ if (!signal) {
+ // Same-device moves stay atomic and do not copy bytes.
+ this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)
+ await beforeCommit?.()
+ await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
+ return
+ }
+ return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {
+ beforeCommit,
+ cacheKey: srcSpace.task!.cacheKey,
+ crossDevice: true,
+ onProgress: (bytes) => this.updateProgress(srcSpace, bytes),
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),
+ overwrite,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ async delete(
+ user: UserModel,
+ space: SpaceEnv,
+ trashFile: string,
+ isDir: boolean,
+ signal: AbortSignal | undefined,
+ prepareDestination: () => Promise<void>
+ ): Promise<SourceCleanupError | undefined> {
+ await this.initializeTaskProps(space, isDir)
+ if (!signal) {
+ // Moving to trash on the same device stays atomic and does not copy bytes.
+ this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)
+ await prepareDestination()
+ await moveFiles(space.realPath, trashFile, true)
+ return
+ }
+ return this.moveAbortable(space.realPath, trashFile, {
+ beforeCommit: prepareDestination,
+ cacheKey: space.task!.cacheKey,
+ crossDevice: true,
+ onProgress: (bytes) => this.updateProgress(space, bytes),
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),
+ overwrite: true,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ private async initializeTaskProps(space: SpaceEnv, isDir: boolean): Promise<void> {
+ const metrics = isDir ? await countDirEntriesAndSize(space.realPath) : { size: await fileSize(space.realPath) }
+ space.task!.props = {
+ ...space.task!.props,
+ ...metrics,
+ progress: 1,
+ size: 0,
+ totalSize: metrics.size
+ }
+ }
+
+ private updateProgress(space: SpaceEnv, bytes: number): void {
+ const props = space.task!.props
+ props.size = Math.min((props.size || 0) + bytes, props.totalSize || Number.MAX_SAFE_INTEGER)
+ if (props.totalSize) {
+ props.progress = Math.min((100 * props.size) / props.totalSize, 100)
+ }
+ }
+
+ private prepareTaskDestination(
+ srcSpace: SpaceEnv,
+ dstSpace: SpaceEnv,
+ overwrite: boolean,
+ deleteDestination: () => Promise<void>
+ ): (() => Promise<void>) | undefined {
+ if (!overwrite || srcSpace.realPath.toLowerCase() === dstSpace.realPath.toLowerCase()) return
+ return async () => {
+ if (await isPathExists(dstSpace.realPath)) {
+ await deleteDestination()
+ }
+ }
+ }
+
+ private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {
+ FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)
+ }
+
+ private async copyAbortable(srcPath: string, dstPath: string, options: FileTaskCopyTaskOptions): Promise<void> {
+ const {
+ beforeCommit,
+ cacheKey,
+ onProgress,
+ onTransferStart,
+ overwrite = false,
+ preserveTimestamps = true,
+ recursive = true,
+ signal,
+ stagingDir = path.dirname(dstPath)
+ } = options
+ const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)
+ await fs.mkdir(stagingDir, { recursive: true })
+ const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)
+
+ if (copyDirectlyToDestination) {
+ let transferStarted = false
+ try {
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ signal.throwIfAborted()
+ await this.prepareDestination(dstPath, overwrite)
+ onTransferStart?.(dstPath)
+ transferStarted = true
+ await this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)
+ } catch (e) {
+ if (transferStarted) {
+ await this.cleanupAfterFailure(dstPath)
+ }
+ throw e
+ }
+ return
+ }
+
+ try {
+ onTransferStart?.(temporaryPath)
+ await this.copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal, onProgress)
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ await this.prepareDestination(dstPath, overwrite)
+ await fs.rename(temporaryPath, dstPath)
+ } catch (e) {
+ await this.cleanupAfterFailure(temporaryPath)
+ throw e
+ }
+ }
+
+ private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {
+ const { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite = false, signal, stagingDir } = options
+ const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))
+ if (!crossDevice) {
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ await moveFiles(srcPath, dstPath, overwrite)
+ return
+ }
+ await this.copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite, signal, stagingDir })
+ try {
+ await removeFiles(srcPath)
+ } catch (cause) {
+ return new SourceCleanupError(srcPath, dstPath, { cause })
+ }
+ }
+
+ private async copyEntry(
+ srcPath: string,
+ dstPath: string,
+ recursive: boolean,
+ preserveTimestamps: boolean,
+ signal: AbortSignal,
+ onProgress?: (bytes: number) => void
+ ): Promise<void> {
+ signal.throwIfAborted()
+ const stats = await fs.lstat(srcPath)
+ if (stats.isDirectory()) {
+ await fs.mkdir(dstPath)
+ if (recursive) {
+ for (const entry of await fs.readdir(srcPath)) {
+ await this.copyEntry(path.join(srcPath, entry), path.join(dstPath, entry), true, preserveTimestamps, signal, onProgress)
+ }
+ }
+ } else if (stats.isSymbolicLink()) {
+ await fs.symlink(await fs.readlink(srcPath), dstPath)
+ } else {
+ const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ if (onProgress) {
+ const progress = new Transform({
+ transform(chunk: Buffer, _encoding, callback) {
+ onProgress(chunk.length)
+ callback(null, chunk)
+ }
+ })
+ await pipeline(src, progress, dst, { signal })
+ } else {
+ await pipeline(src, dst, { signal })
+ }
+ }
+ if (!stats.isSymbolicLink()) {
+ await fs.chmod(dstPath, stats.mode)
+ if (preserveTimestamps) {
+ await fs.utimes(dstPath, stats.atime, stats.mtime)
+ }
+ }
+ }
+
+ private async prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {
+ if (!(await isPathExists(dstPath))) return
+ if (!overwrite) {
+ throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })
+ }
+ await removeFiles(dstPath)
+ }
+
+ private async cleanupAfterFailure(rPath: string): Promise<void> {
+ try {
+ await removeFiles(rPath)
+ } catch {
+ // Cleanup is best-effort and must not replace the transfer error.
+ }
+ }
+}
diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts
new file mode 100644
index 00000000..9b00d02b
--- /dev/null
+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts
@@ -0,0 +1,181 @@
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { CACHE_TASK_TTL } from '../../constants/cache'
+import { FILE_OPERATION } from '../../constants/operations'
+import { FileTaskEvent } from '../../events/file-events'
+import { FileTaskStatus } from '../../models/file-task'
+import * as filesUtils from '../../utils/files'
+import * as tasksUtils from '../../utils/tasks'
+import { FilesTasksWatcher } from './files-tasks-watcher.service'
+import { Mock } from 'vitest'
+
+describe(FilesTasksWatcher.name, () => {
+ let service: FilesTasksWatcher
+ let cacheStore: Map<string, any>
+ let cache: {
+ get: Mock
+ set: Mock
+ }
+
+ const flushPromises = async () => {
+ await Promise.resolve()
+ await Promise.resolve()
+ }
+
+ const makeSpace = (cacheKey = 'task-1', props: Record<string, any> = {}) =>
+ ({
+ url: 'files/personal/source.txt',
+ task: { cacheKey, props }
+ }) as any
+
+ beforeEach(() => {
+ cacheStore = new Map()
+ cache = {
+ get: vi.fn(async (key: string) => cacheStore.get(key)),
+ set: vi.fn(async (key: string, value: unknown) => {
+ cacheStore.set(key, value)
+ return true
+ })
+ }
+ service = new FilesTasksWatcher(cache as unknown as Cache)
+ })
+
+ afterEach(() => {
+ service.onModuleDestroy()
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('subscribes to task watch events and unsubscribes on module destroy', () => {
+ const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')
+
+ service.onModuleDestroy()
+
+ expect(FileTaskEvent.listenerCount('startWatch')).toBe(listenersBeforeDestroy - 1)
+ })
+
+ it('returns file or directory metrics for a watched path', async () => {
+ vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)
+ vi.spyOn(tasksUtils, 'countDirEntriesAndSize').mockResolvedValueOnce({ files: 2, directories: 1, size: 84 })
+
+ await expect(service.getPathProps('/files/report.txt')).resolves.toEqual({ size: 42 })
+ await expect(service.getPathProps('/files/archive')).resolves.toEqual({ files: 2, directories: 1, size: 84 })
+ })
+
+ it('starts a download watcher with the published task metadata', async () => {
+ const space = makeSpace()
+ const updateTask = vi.spyOn(service as any, 'updateTask').mockResolvedValue(undefined)
+ const updateDownloadTask = vi.spyOn(service as any, 'updateDownloadTask').mockResolvedValue(undefined)
+ const watchTask = vi.spyOn(service as any, 'watchTask').mockImplementation((_cacheKey: string, update: () => Promise<void>) => void update())
+
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, '/files/report.txt', '/tmp/report.part')
+ await flushPromises()
+
+ expect(updateTask).toHaveBeenCalledWith('task-1', {}, { name: 'report.txt', path: 'files/personal' })
+ expect(watchTask).toHaveBeenCalledWith('task-1', expect.any(Function))
+ expect(updateDownloadTask).toHaveBeenCalledWith(space, '/tmp/report.part', '/files/report.txt')
+ })
+
+ it('does not start duplicate watchers for the same task', () => {
+ const space = makeSpace()
+ const watchTask = vi.spyOn(service as any, 'watchTask')
+ ;(service as any).tasksWatcher['task-1'] = setInterval(() => undefined, 1000)
+ ;(service as any).startWatch(space, FILE_OPERATION.COPY, '/files/report.txt')
+
+ expect(watchTask).not.toHaveBeenCalled()
+ })
+
+ it('does not overlap interval updates', async () => {
+ vi.useFakeTimers()
+ let finishUpdate!: () => void
+ const update = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ finishUpdate = resolve
+ })
+ )
+
+ ;(service as any).watchTask('task-1', update)
+ await vi.advanceTimersByTimeAsync(2000)
+
+ expect(update).toHaveBeenCalledTimes(1)
+
+ finishUpdate()
+ await flushPromises()
+ await vi.advanceTimersByTimeAsync(1000)
+
+ expect(update).toHaveBeenCalledTimes(2)
+ })
+
+ it('merges active task metadata and properties in the cache', async () => {
+ cacheStore.set('task-1', {
+ name: 'source.txt',
+ path: 'files/personal',
+ props: { progress: 1, totalSize: 100 },
+ status: FileTaskStatus.PENDING
+ })
+
+ await (service as any).updateTask('task-1', { progress: 40, size: 40 }, { name: 'destination.txt' })
+
+ expect(cache.set).toHaveBeenCalledWith(
+ 'task-1',
+ {
+ name: 'destination.txt',
+ path: 'files/personal',
+ props: { progress: 40, size: 40, totalSize: 100 },
+ status: FileTaskStatus.PENDING
+ },
+ CACHE_TASK_TTL
+ )
+ })
+
+ it.each([undefined, FileTaskStatus.SUCCESS])('stops watching when the cached task is missing or inactive', async (status) => {
+ if (status !== undefined) {
+ cacheStore.set('task-1', { props: {}, status })
+ }
+ const stopWatch = vi.spyOn(service, 'stopWatch')
+
+ await (service as any).updateTask('task-1', { size: 10 })
+
+ expect(stopWatch).toHaveBeenCalledWith('task-1')
+ expect(cache.set).not.toHaveBeenCalled()
+ })
+
+ it('waits for pending updates and prevents cache writes during finalization', async () => {
+ let resolveGet!: (value: unknown) => void
+ cache.get.mockReturnValueOnce(
+ new Promise((resolve) => {
+ resolveGet = resolve
+ })
+ )
+ const update = (service as any).updateTask('task-1', { size: 10 })
+ await flushPromises()
+
+ const finalization = service.beginFinalization('task-1')
+ resolveGet({ props: {}, status: FileTaskStatus.PENDING })
+ await Promise.all([update, finalization])
+
+ expect(cache.set).not.toHaveBeenCalled()
+ expect((service as any).finalizingTasks.has('task-1')).toBe(true)
+
+ service.endFinalization('task-1')
+ expect((service as any).finalizingTasks.has('task-1')).toBe(false)
+ })
+
+ it('falls back to the published file and updates download progress', async () => {
+ const space = makeSpace('task-1', { size: 0, totalSize: 200 })
+ cacheStore.set('task-1', {
+ props: { size: 0, totalSize: 200 },
+ status: FileTaskStatus.PENDING
+ })
+ vi.spyOn(filesUtils, 'isPathExists').mockImplementation(async (rPath: string) => rPath === '/files/report.txt')
+ vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false)
+ vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(50)
+
+ await (service as any).updateDownloadTask(space, '/tmp/report.part', '/files/report.txt')
+
+ expect(filesUtils.fileSize).toHaveBeenCalledWith('/files/report.txt')
+ expect(space.task.props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })
+ expect(cacheStore.get('task-1').props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })
+ })
+})
diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts
new file mode 100644
index 00000000..16d09d1d
--- /dev/null
+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts
@@ -0,0 +1,201 @@
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'
+import { Cache } from '../../../../infrastructure/cache/cache.service'
+import { SpaceEnv } from '../../../spaces/models/space-env.model'
+import { CACHE_TASK_TTL } from '../../constants/cache'
+import { FILE_OPERATION } from '../../constants/operations'
+import { FileTaskEvent } from '../../events/file-events'
+import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'
+import { dirName, fileName, fileSize, isPathExists, isPathIsDir } from '../../utils/files'
+import { countDirEntriesAndSize } from '../../utils/tasks'
+
+@Injectable()
+export class FilesTasksWatcher implements OnModuleDestroy {
+ private readonly logger = new Logger(FilesTasksWatcher.name)
+ private readonly watchInterval = 1000
+ private tasksWatcher: Record<string, NodeJS.Timeout> = {}
+ private pendingTaskUpdates = new Map<string, Promise<void>>()
+ private finalizingTasks = new Set<string>()
+
+ constructor(private readonly cache: Cache) {
+ FileTaskEvent.on('startWatch', this.onStartWatch)
+ }
+
+ onModuleDestroy(): void {
+ FileTaskEvent.off('startWatch', this.onStartWatch)
+ for (const cacheKey of Object.keys(this.tasksWatcher)) {
+ this.stopWatch(cacheKey)
+ }
+ }
+
+ async beginFinalization(cacheKey: string): Promise<void> {
+ this.finalizingTasks.add(cacheKey)
+ this.stopWatch(cacheKey)
+ await this.pendingTaskUpdates.get(cacheKey)?.catch((e: Error) => this.logger.error({ tag: this.beginFinalization.name, msg: `${e}` }))
+ }
+
+ endFinalization(cacheKey: string): void {
+ this.finalizingTasks.delete(cacheKey)
+ }
+
+ stopWatch(cacheKey: string): void {
+ if (!(cacheKey in this.tasksWatcher)) return
+ clearInterval(this.tasksWatcher[cacheKey])
+ delete this.tasksWatcher[cacheKey]
+ }
+
+ async getPathProps(rPath: string): Promise<Pick<FileTaskProps, 'files' | 'directories' | 'size'>> {
+ return (await isPathIsDir(rPath)) ? countDirEntriesAndSize(rPath) : { size: await fileSize(rPath) }
+ }
+
+ private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {
+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)
+ }
+
+ private startWatch(
+ space: SpaceEnv,
+ taskType: FILE_OPERATION,
+ rPath: string,
+ taskPath?: string,
+ taskName = fileName(rPath),
+ publishedPath?: string
+ ): void {
+ const taskContext = space.task
+ if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return
+ const { cacheKey, props } = taskContext
+ this.updateTask(cacheKey, props, {
+ name: taskName,
+ path: taskPath
+ }).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
+ switch (taskType) {
+ case FILE_OPERATION.COMPRESS:
+ this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))
+ return
+ case FILE_OPERATION.DECOMPRESS:
+ this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))
+ return
+ case FILE_OPERATION.DOWNLOAD:
+ this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))
+ return
+ case FILE_OPERATION.COPY:
+ case FILE_OPERATION.DELETE:
+ case FILE_OPERATION.MOVE:
+ this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))
+ return
+ default:
+ this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })
+ }
+ }
+
+ private watchTask(cacheKey: string, update: () => Promise<void>): void {
+ // Directory scans may exceed the interval, so ticks use exhaust semantics.
+ let updateInProgress = false
+ const watcher = setInterval(() => {
+ if (updateInProgress) return
+ updateInProgress = true
+ void update()
+ .catch((e: Error) => this.logger.error({ tag: this.watchTask.name, msg: `${e}` }))
+ .finally(() => {
+ updateInProgress = false
+ })
+ }, this.watchInterval)
+ watcher.unref()
+ this.tasksWatcher[cacheKey] = watcher
+ }
+
+ private updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {
+ if (this.finalizingTasks.has(cacheKey)) return Promise.resolve()
+ const previousUpdate = this.pendingTaskUpdates.get(cacheKey) ?? Promise.resolve()
+ const update = previousUpdate
+ .catch(() => undefined)
+ .then(async () => {
+ if (this.finalizingTasks.has(cacheKey)) return
+ let fileTask: FileTask = await this.cache.get(cacheKey)
+ if (!fileTask) {
+ this.stopWatch(cacheKey)
+ return
+ }
+ if (!this.isActiveStatus(fileTask.status) || this.finalizingTasks.has(cacheKey)) {
+ this.stopWatch(cacheKey)
+ return
+ }
+ if (task) fileTask = { ...fileTask, ...task }
+ if (props) fileTask.props = { ...fileTask.props, ...props }
+ await this.cache.set(cacheKey, fileTask, CACHE_TASK_TTL)
+ })
+ this.pendingTaskUpdates.set(cacheKey, update)
+ return update.finally(() => {
+ if (this.pendingTaskUpdates.get(cacheKey) === update) this.pendingTaskUpdates.delete(cacheKey)
+ })
+ }
+
+ private isActiveStatus(status: FileTaskStatus): boolean {
+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
+ }
+
+ private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
+ try {
+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
+ if (size === undefined) return
+ space.task.props.size = size
+ await this.updateTask(space.task.cacheKey, space.task.props)
+ } catch (e) {
+ this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
+ this.stopWatch(space.task.cacheKey)
+ }
+ }
+
+ private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
+ try {
+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)
+ if (props === undefined) return
+ space.task.props = props
+ await this.updateTask(space.task.cacheKey, space.task.props)
+ } catch (e) {
+ this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
+ this.stopWatch(space.task.cacheKey)
+ }
+ }
+
+ private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
+ try {
+ if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return
+ await this.updateTask(space.task.cacheKey, space.task.props)
+ } catch (e) {
+ this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
+ this.stopWatch(space.task.cacheKey)
+ }
+ }
+
+ private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {
+ const isDir = await this.readWatchedPath(rPath, publishedPath, isPathIsDir)
+ if (isDir === undefined) return false
+ if (isDir) {
+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)
+ if (props === undefined) return false
+ space.task.props = { ...space.task.props, ...props }
+ } else {
+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
+ if (size === undefined) return false
+ space.task.props.size = size
+ }
+ if (space.task.props.totalSize) {
+ space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize
+ }
+ return true
+ }
+
+ private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {
+ if (!publishedPath) return read(rPath)
+ if (await isPathExists(rPath)) {
+ try {
+ const value = await read(rPath)
+ // The temporary path may be moved atomically while it is being read.
+ if (await isPathExists(rPath)) return value
+ } catch (e) {
+ // Retry the published path only when the temporary path disappeared during publication.
+ if (e?.code !== 'ENOENT') throw e
+ }
+ }
+ if (await isPathExists(publishedPath)) return read(publishedPath)
+ }
+}
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index 6ad551c9..a3841ec0 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -244,7 +244,7 @@ export function copyFileContent(srcPath: string, dstPath: string): Promise<void>
return writeFromStream(dstPath, srcStream)
}
-async function walkDir(
+export async function walkDir(
rPath: string,
onEntry: (entry: Dirent, entryPath: string) => Promise<void> | void,
errors?: Record<string, string>
@@ -291,25 +291,6 @@ export async function dirListFileNames(rPath: string): Promise<string[]> {
return (await fs.readdir(rPath)).map((path: string) => fileName(path))
}
-export async function countDirEntries(rPath: string): Promise<{ files: number; directories: number }> {
- const entriesCount = { files: 0, directories: 0 }
- const ignoredErrors: Record<string, string> = {}
-
- await walkDir(
- rPath,
- (entry: Dirent) => {
- if (entry.isDirectory()) {
- entriesCount.directories++
- } else {
- entriesCount.files++
- }
- },
- ignoredErrors
- )
-
- return entriesCount
-}
-
export async function dirHasChildren(rPath: string, mustContainsDirs = true): Promise<boolean> {
for await (const file of await fs.opendir(rPath)) {
if (mustContainsDirs) {
diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.ts
index 71f30d77..dbd566c8 100644
--- a/backend/src/applications/files/utils/tasks.spec.ts
+++ b/backend/src/applications/files/utils/tasks.spec.ts
@@ -1,22 +1,19 @@
-import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import fs, { mkdtemp, rm, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { FILE_OPERATION } from '../constants/operations'
-import * as filesUtils from './files'
-import { copyAbortable, createTaskTemporaryDir, isTaskCancellable, moveAbortable, SourceCleanupError, taskTemporaryPath } from './tasks'
+import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath } from './tasks'
-describe('abortable file tasks', () => {
+describe('file task utilities', () => {
const cacheKey = 'ftask-7-task-id'
let tmpDir: string
let srcPath: string
let dstPath: string
- let stagingDir: string
beforeEach(async () => {
tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))
srcPath = path.join(tmpDir, 'source.txt')
dstPath = path.join(tmpDir, 'destination.txt')
- stagingDir = path.join(tmpDir, 'tasks')
})
afterEach(async () => {
@@ -24,11 +21,6 @@ describe('abortable file tasks', () => {
vi.restoreAllMocks()
})
- const mockCrossDevice = () => {
- vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)
- vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
- }
-
it('marks intrinsically abortable operations as cancellable', async () => {
await expect(isTaskCancellable(FILE_OPERATION.COPY, '/source')).resolves.toBe(true)
await expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD, '/source')).resolves.toBe(true)
@@ -57,139 +49,4 @@ describe('abortable file tasks', () => {
expect(path.basename(directoryPath)).toBe(`.${cacheKey}-archive`)
expect((await fs.stat(directoryPath)).isDirectory()).toBe(true)
})
-
- it('publishes a copied file only after the commit hook', async () => {
- const signal = new AbortController().signal
- let temporaryPath = ''
- await writeFile(srcPath, 'content')
-
- await copyAbortable(srcPath, dstPath, {
- beforeCommit: async () => {
- await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
- await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')
- },
- cacheKey,
- onTransferStart: (value) => {
- temporaryPath = value
- },
- signal,
- stagingDir
- })
-
- expect(path.dirname(temporaryPath)).toBe(stagingDir)
- expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
- await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })
- })
-
- it('cleans the temporary destination when copy is cancelled', async () => {
- const controller = new AbortController()
- await writeFile(srcPath, 'content')
- controller.abort()
-
- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal: controller.signal, stagingDir })).rejects.toMatchObject({
- name: 'AbortError'
- })
-
- await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
- await expect(readdir(stagingDir)).resolves.toEqual([])
- })
-
- it('copies directly to a destination on another device', async () => {
- const signal = new AbortController().signal
- let watchedPath = ''
- let destinationPrepared = false
- await writeFile(srcPath, 'content')
- mockCrossDevice()
-
- await copyAbortable(srcPath, dstPath, {
- beforeCommit: async () => {
- destinationPrepared = true
- },
- cacheKey,
- onTransferStart: (value) => {
- expect(destinationPrepared).toBe(true)
- watchedPath = value
- },
- signal,
- stagingDir
- })
-
- expect(watchedPath).toBe(dstPath)
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
- await expect(readdir(stagingDir)).resolves.toEqual([])
- })
-
- it('keeps an existing direct-copy destination when overwrite is rejected', async () => {
- const signal = new AbortController().signal
- await writeFile(srcPath, 'new content')
- await writeFile(dstPath, 'existing content')
- mockCrossDevice()
-
- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toMatchObject({ code: 'EEXIST' })
-
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
- })
-
- it('does not prepare a direct-copy destination when already cancelled', async () => {
- const controller = new AbortController()
- const beforeCommit = vi.fn()
- await writeFile(srcPath, 'new content')
- await writeFile(dstPath, 'existing content')
- await fs.mkdir(stagingDir)
- controller.abort()
- mockCrossDevice()
-
- await expect(
- copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, overwrite: true, signal: controller.signal, stagingDir })
- ).rejects.toMatchObject({ name: 'AbortError' })
-
- expect(beforeCommit).not.toHaveBeenCalled()
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
- })
-
- it('preserves a direct-copy error when destination cleanup also fails', async () => {
- const signal = new AbortController().signal
- const transferError = Object.assign(new Error('transfer failed'), { code: 'EIO' })
- await fs.mkdir(stagingDir)
- vi.spyOn(fs, 'lstat').mockImplementation(async (rPath) => {
- if (rPath === stagingDir) return { dev: 1 } as any
- throw transferError
- })
- vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
- vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(new Error('cleanup failed'))
-
- await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toBe(transferError)
- })
-
- it('copies then removes the source for a cross-device move', async () => {
- const signal = new AbortController().signal
- await writeFile(srcPath, 'content')
-
- await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })
-
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
- await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
- await expect(readdir(stagingDir)).resolves.toEqual([])
- })
-
- it('reports source cleanup failure after keeping the published destination', async () => {
- const signal = new AbortController().signal
- const error = new Error('source cleanup failed')
- await writeFile(srcPath, 'content')
- vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)
-
- const cleanupError = await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })
-
- expect(cleanupError).toBeInstanceOf(SourceCleanupError)
- expect(cleanupError).toMatchObject({
- cause: error,
- dstPath,
- message: 'Destination was published but the source could not be removed',
- name: SourceCleanupError.name,
- srcPath
- })
- await expect(readFile(srcPath, 'utf8')).resolves.toBe('content')
- await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
- })
})
diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.ts
index b38748e8..e159c448 100644
--- a/backend/src/applications/files/utils/tasks.ts
+++ b/backend/src/applications/files/utils/tasks.ts
@@ -1,42 +1,9 @@
-import { createReadStream, createWriteStream } from 'node:fs'
+import { Dirent } from 'node:fs'
import fs from 'node:fs/promises'
import path from 'node:path'
-import { pipeline } from 'node:stream/promises'
-import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'
import { FILE_OPERATION } from '../constants/operations'
-import { isPathExists, moveFiles, removeFiles } from './files'
-
-export interface TransferOptions {
- // Runs immediately before the destination can be modified or published.
- beforeCommit?: () => Promise<void>
- // Identifies the owning task so cleanup can preserve its staging entry.
- cacheKey: string
- // Allows callers that already resolved the filesystem topology to skip another stat.
- crossDevice?: boolean
- // Provides the path whose progress must be watched during the transfer.
- onTransferStart?: (temporaryPath: string) => void
- overwrite?: boolean
- signal: AbortSignal
- // Preferred staging directory. Atomic publication requires it to share the destination filesystem.
- stagingDir?: string
-}
-
-export interface CopyTaskOptions extends TransferOptions {
- preserveTimestamps?: boolean
- recursive?: boolean
-}
-
-// The destination is committed, but the obsolete source still requires manual or deferred cleanup.
-export class SourceCleanupError extends Error {
- constructor(
- readonly srcPath: string,
- readonly dstPath: string,
- options: ErrorOptions
- ) {
- super('Destination was published but the source could not be removed', options)
- this.name = SourceCleanupError.name
- }
-}
+import { isPathExists, walkDir } from './files'
+import type { FileTaskProps } from '../models/file-task'
export function taskTemporaryPrefix(cacheKey: string): string {
return `.${cacheKey}-`
@@ -53,33 +20,6 @@ export async function createTaskTemporaryDir(parentPath: string, cacheKey: strin
return temporaryPath
}
-async function copyEntry(srcPath: string, dstPath: string, recursive: boolean, preserveTimestamps: boolean, signal: AbortSignal): Promise<void> {
- signal.throwIfAborted()
- const stats = await fs.lstat(srcPath)
- if (stats.isDirectory()) {
- await fs.mkdir(dstPath)
- if (recursive) {
- for (const entry of await fs.readdir(srcPath)) {
- await copyEntry(path.join(srcPath, entry), path.join(dstPath, entry), true, preserveTimestamps, signal)
- }
- }
- } else if (stats.isSymbolicLink()) {
- await fs.symlink(await fs.readlink(srcPath), dstPath)
- } else {
- await pipeline(
- createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK }),
- createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK }),
- { signal }
- )
- }
- if (!stats.isSymbolicLink()) {
- await fs.chmod(dstPath, stats.mode)
- if (preserveTimestamps) {
- await fs.utimes(dstPath, stats.atime, stats.mtime)
- }
- }
-}
-
async function existingParentPath(dstPath: string): Promise<string> {
let parentPath = path.dirname(dstPath)
while (!(await isPathExists(parentPath))) {
@@ -115,89 +55,27 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, d
}
}
-async function prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {
- if (!(await isPathExists(dstPath))) return
- if (!overwrite) {
- throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })
- }
- await removeFiles(dstPath)
-}
-
-async function cleanupAfterFailure(rPath: string): Promise<void> {
- try {
- await removeFiles(rPath)
- } catch {
- // Cleanup is best-effort and must not replace the transfer error.
- }
-}
-
-export async function copyAbortable(srcPath: string, dstPath: string, options: CopyTaskOptions): Promise<void> {
- const {
- beforeCommit,
- cacheKey,
- onTransferStart,
- overwrite = false,
- preserveTimestamps = true,
- recursive = true,
- signal,
- stagingDir = path.dirname(dstPath)
- } = options
- const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)
- await fs.mkdir(stagingDir, { recursive: true })
- const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)
-
- // A staging file cannot be renamed atomically across filesystems. In that case,
- // copy directly to the destination and remove partial data on handled failures.
- // beforeCommit may already have moved an overwritten destination to trash when
- // cancellation occurs; this recoverable side effect is accepted for external filesystems.
- if (copyDirectlyToDestination) {
- let transferStarted = false
- try {
- // Check before modifying the destination, then again before starting the long direct transfer.
- signal.throwIfAborted()
- await beforeCommit?.()
- signal.throwIfAborted()
- await prepareDestination(dstPath, overwrite)
- onTransferStart?.(dstPath)
- transferStarted = true
- await copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal)
- } catch (e) {
- if (transferStarted) {
- await cleanupAfterFailure(dstPath)
+export async function countDirEntriesAndSize(rPath: string): Promise<Pick<FileTaskProps, 'files' | 'directories' | 'size'>> {
+ const entriesCount = { files: 0, directories: 0, size: 0 }
+ const ignoredErrors: Record<string, string> = {}
+
+ await walkDir(
+ rPath,
+ async (entry: Dirent, entryPath: string) => {
+ if (entry.isDirectory()) {
+ entriesCount.directories++
+ } else {
+ entriesCount.files++
+ if (!entry.isFile()) return
+ try {
+ entriesCount.size += (await fs.stat(entryPath)).size
+ } catch {
+ // ignore
+ }
}
- throw e
- }
- return
- }
-
- // Same-filesystem staging keeps the destination untouched until the final rename.
- try {
- onTransferStart?.(temporaryPath)
- await copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal)
- signal.throwIfAborted()
- await beforeCommit?.()
- await prepareDestination(dstPath, overwrite)
- await fs.rename(temporaryPath, dstPath)
- } catch (e) {
- await cleanupAfterFailure(temporaryPath)
- throw e
- }
-}
+ },
+ ignoredErrors
+ )
-export async function moveAbortable(srcPath: string, dstPath: string, options: TransferOptions): Promise<SourceCleanupError | undefined> {
- const { beforeCommit, cacheKey, onTransferStart, overwrite = false, signal, stagingDir } = options
- const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))
- if (!crossDevice) {
- signal.throwIfAborted()
- await beforeCommit?.()
- await moveFiles(srcPath, dstPath, overwrite)
- return
- }
- await copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, onTransferStart, overwrite, signal, stagingDir })
- try {
- await removeFiles(srcPath)
- } catch (cause) {
- // Publication is the commit point. The caller must update its persistent state before reporting this cleanup failure.
- return new SourceCleanupError(srcPath, dstPath, { cause })
- }
+ return entriesCount
}