Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): add cancellable copy, move and delete tasks

johavene23151ea16db
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 7d18ad46..abee5c11 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -25,6 +25,7 @@ import { SendFile } from '../utils/send-file'
import * as unzipUtils from '../utils/unzip-file'
import * as untarUtils from '../utils/untar-file'
import * as filesUtils from '../utils/files'
+import * as taskUtils from '../utils/tasks'
import { FilesLockManager } from './files-lock-manager.service'
import { FilesManager } from './files-manager.service'
import { FilesQueries } from './files-queries.service'
@@ -72,7 +73,7 @@ describe(FilesManager.name, () => {
quotaIsExceeded: false,
storageQuota: null,
willExceedQuota: vi.fn().mockReturnValue(false),
- task: { cacheKey: '', props: {} },
+ task: undefined,
...overrides
}) as any
@@ -80,6 +81,11 @@ describe(FilesManager.name, () => {
vi.mocked(filesUtils.isPathExists).mockImplementation(async (p: string) => (p in values ? values[p] : fallback))
}
+ const prepareFileTransfer = (srcPath: string, dstPath: string, dstExists = false) => {
+ setPathExists({ [srcPath]: true, [path.dirname(dstPath)]: true, [dstPath]: dstExists }, false)
+ vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)
+ }
+
const makeTrashSpace = (overrides: Record<string, any> = {}) =>
makeSpace({
repository: 'trash',
@@ -121,6 +127,8 @@ describe(FilesManager.name, () => {
expect(filesUtils.copyFiles).not.toHaveBeenCalled()
expect(filesUtils.moveFiles).not.toHaveBeenCalled()
expect(filesUtils.removeFiles).not.toHaveBeenCalled()
+ expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
+ expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
expect(filesLockManager.create).not.toHaveBeenCalled()
expect(filesLockManager.createOrRefresh).not.toHaveBeenCalled()
expect(filesLockManager.checkConflicts).not.toHaveBeenCalled()
@@ -184,6 +192,19 @@ describe(FilesManager.name, () => {
vi.spyOn(filesUtils, 'dirSize').mockResolvedValue([123, {}] as any)
vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/unique-path.txt')
vi.spyOn(filesUtils, 'uniqueDatedFilePath').mockResolvedValue({ isDir: false, path: '/trash/file-2026.txt' })
+ vi.spyOn(taskUtils, 'copyAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {
+ options.onTransferStart?.('/tmp/task-transfer')
+ await options.beforeCommit?.()
+ })
+ vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue('/data/users/john/tmp/tasks/.task-d-archive')
+ vi.spyOn(taskUtils, 'moveAbortable').mockImplementation(async (_srcPath, _dstPath, options) => {
+ options.onTransferStart?.('/tmp/task-transfer')
+ await options.beforeCommit?.()
+ return undefined
+ })
+ vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>
+ path.join(parentPath, `.${cacheKey}-${path.basename(name)}`)
+ )
vi.spyOn(filesUtils, 'getMimeType').mockReturnValue('image-png')
vi.spyOn(spacesPermsUtils, 'canAccessToSpace').mockReturnValue(true)
vi.spyOn(spacesPermsUtils, 'haveSpaceEnvPermissions').mockReturnValue(true)
@@ -791,6 +812,7 @@ describe(FilesManager.name, () => {
expect(filesLockManager.checkConflicts).toHaveBeenCalledWith(dst.dbFile, DEPTH.RESOURCE, { userId: 7, lockTokens: undefined })
expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false, false)
+ expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: dst.realPath })
})
@@ -824,10 +846,112 @@ describe(FilesManager.name, () => {
await service.copyMove(user, src, dst, true)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/src-base/src.txt', '/dst-base/dst.txt', false)
+ expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: src, action: ACTION.DELETE_PERMANENTLY, rPath: '/src-base/src.txt' })
expect(emitSpy).toHaveBeenCalledWith('event', { user, space: dst, action: ACTION.ADD, rPath: '/dst-base/dst.txt' })
})
+
+ it('should update the database before reporting an abortable move source cleanup failure', async () => {
+ const src = makeSpace({
+ url: 'files/source/src.txt',
+ realPath: '/src-base/src.txt',
+ dbFile: { ownerId: 7, path: 'src.txt', inTrash: false },
+ task: { cacheKey: 'task-move', props: {} }
+ })
+ const dst = makeSpace({
+ url: 'files/destination/dst.txt',
+ realPath: '/dst-base/dst.txt',
+ dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }
+ })
+ const signal = new AbortController().signal
+ const cleanupError = new taskUtils.SourceCleanupError(src.realPath, dst.realPath, { cause: new Error('cleanup failed') })
+ prepareFileTransfer(src.realPath, dst.realPath)
+ vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)
+
+ await expect(service.copyMove(user, src, dst, true, false, false, undefined, signal)).rejects.toBe(cleanupError)
+
+ expect(taskUtils.moveAbortable).toHaveBeenCalledWith(
+ src.realPath,
+ dst.realPath,
+ expect.objectContaining({
+ cacheKey: 'task-move',
+ crossDevice: true,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ )
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ expect(filesQueries.moveFiles).toHaveBeenCalledWith(src.dbFile, dst.dbFile, false)
+ })
+
+ it('should use the regular move path for a non-cancellable move task', async () => {
+ const src = makeSpace({
+ realPath: '/data/users/john/files/src.txt',
+ dbFile: { ownerId: 7, path: 'src.txt', inTrash: false },
+ task: { cacheKey: 'task-move', props: {} }
+ })
+ const dst = makeSpace({
+ realPath: '/data/users/john/files/dst.txt',
+ dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }
+ })
+ prepareFileTransfer(src.realPath, dst.realPath)
+
+ await service.copyMove(user, src, dst, true)
+
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, false)
+ expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ })
+
+ it('should preserve the regular overwrite path outside a task context', async () => {
+ const src = makeSpace({
+ realPath: '/data/users/john/files/src.txt',
+ dbFile: { ownerId: 7, path: 'src.txt', inTrash: false }
+ })
+ const dst = makeSpace({
+ realPath: '/data/users/john/files/dst.txt',
+ dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }
+ })
+ prepareFileTransfer(src.realPath, dst.realPath, true)
+ const deleteSpy = vi.spyOn(service, 'delete').mockResolvedValueOnce(undefined)
+
+ await service.copyMove(user, src, dst, false, true)
+
+ expect(deleteSpy).toHaveBeenCalledWith(user, dst)
+ expect(filesUtils.copyFiles).toHaveBeenCalledWith(src.realPath, dst.realPath, true, false)
+ expect(taskUtils.copyAbortable).not.toHaveBeenCalled()
+ expect(deleteSpy.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.copyFiles).mock.invocationCallOrder[0])
+ })
+
+ it('should stage an overwrite before deleting the existing destination', async () => {
+ const src = makeSpace({
+ realPath: '/data/users/john/files/src.txt',
+ dbFile: { ownerId: 7, path: 'src.txt', inTrash: false },
+ task: { cacheKey: 'task-copy', props: {} }
+ })
+ const dst = makeSpace({
+ realPath: '/data/users/john/files/dst.txt',
+ dbFile: { ownerId: 7, path: 'dst.txt', inTrash: false }
+ })
+ prepareFileTransfer(src.realPath, dst.realPath, true)
+ const deleteSpy = vi.spyOn(service, 'delete').mockResolvedValueOnce(undefined)
+ const signal = new AbortController().signal
+
+ await service.copyMove(user, src, dst, false, true, false, undefined, signal)
+
+ expect(taskUtils.copyAbortable).toHaveBeenCalledWith(
+ src.realPath,
+ dst.realPath,
+ expect.objectContaining({
+ beforeCommit: expect.any(Function),
+ cacheKey: 'task-copy',
+ overwrite: true,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ )
+ expect(deleteSpy).toHaveBeenCalledWith(user, dst)
+ })
})
describe('delete', () => {
@@ -858,6 +982,49 @@ describe(FilesManager.name, () => {
expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/files/no-trash.txt')
expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, true)
})
+
+ it('should update the database before reporting an abortable delete source cleanup failure', async () => {
+ const space = makeSpace({
+ realPath: '/data/users/john/files/document.txt',
+ dbFile: { ownerId: 7, path: 'documents/document.txt', inTrash: false },
+ task: { cacheKey: 'task-delete', props: {} }
+ })
+ const trashFile = '/data/users/john/trash/documents/document.txt'
+ const signal = new AbortController().signal
+ const cleanupError = new taskUtils.SourceCleanupError(space.realPath, trashFile, { cause: new Error('cleanup failed') })
+ prepareFileTransfer(space.realPath, trashFile)
+ vi.mocked(taskUtils.moveAbortable).mockResolvedValueOnce(cleanupError)
+
+ await expect(service.delete(user, space, undefined, signal)).rejects.toBe(cleanupError)
+
+ expect(taskUtils.moveAbortable).toHaveBeenCalledWith(
+ space.realPath,
+ trashFile,
+ expect.objectContaining({
+ cacheKey: 'task-delete',
+ onTransferStart: expect.any(Function),
+ overwrite: true,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ )
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, false)
+ })
+
+ it('should keep the regular move path outside a task context', async () => {
+ const space = makeSpace({
+ realPath: '/data/users/john/files/document.txt',
+ dbFile: { ownerId: 7, path: 'documents/document.txt', inTrash: false }
+ })
+ const trashFile = '/data/users/john/trash/documents/document.txt'
+ prepareFileTransfer(space.realPath, trashFile)
+
+ await service.delete(user, space)
+
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(space.realPath, trashFile, true)
+ expect(taskUtils.moveAbortable).not.toHaveBeenCalled()
+ })
})
describe('downloadFromUrl', () => {
@@ -885,7 +1052,6 @@ describe(FilesManager.name, () => {
it('should handle HEAD+GET and emit task watch/event', async () => {
const space = makeSpace({ task: { cacheKey: 'task-1', props: {} } })
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/download.txt')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')
http.axiosRef
.mockResolvedValueOnce({
headers: { 'content-length': '55' },
@@ -906,10 +1072,12 @@ describe(FilesManager.name, () => {
space,
FILE_OPERATION.DOWNLOAD,
'/tmp/download.txt',
- '/data/users/john/tmp/download.txt-download-uuid'
+ '/data/users/john/tmp/tasks/.task-1-download.txt'
)
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', expect.anything(), 0, 55)
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', '/tmp/download.txt')
+ expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')
+ expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
+ expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', expect.anything(), 0, 55)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')
expect(filesLockManager.create).toHaveBeenCalledWith(
user,
expect.objectContaining({ path: 'download.txt' }),
@@ -974,7 +1142,6 @@ describe(FilesManager.name, () => {
it('should archive files and emit events', async () => {
const archive = createArchiveMock()
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
const dto = {
@@ -999,10 +1166,14 @@ describe(FilesManager.name, () => {
space,
FILE_OPERATION.COMPRESS,
'/tmp/archive.tar.gz',
- '/data/users/john/tmp/archive.tar.gz-compress-uuid'
+ '/data/users/john/tmp/tasks/.task-c-archive.tar.gz'
)
- expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', { highWaterMark: expect.any(Number) })
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', '/tmp/archive.tar.gz')
+ expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')
+ expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
+ expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {
+ highWaterMark: expect.any(Number)
+ })
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')
})
it('should allow archive export from trash when compressInDirectory is false', async () => {
@@ -1148,21 +1319,21 @@ describe(FilesManager.name, () => {
const space = makeSpace({ realPath: '/data/users/john/files/archive.zip', task: { cacheKey: 'task-d', props: {} } })
vi.mocked(filesUtils.isPathExists).mockResolvedValueOnce(true).mockResolvedValueOnce(false)
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/data/users/john/files/archive')
- vi.mocked(filesUtils.makeTempDir).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
const unzipSpy = vi.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)
const taskEmitSpy = vi.spyOn(FileTaskEvent, 'emit')
await service.decompress(user, space)
- expect(filesUtils.makeTempDir).toHaveBeenCalledWith('/data/users/john/tmp', 'archive-extract-')
- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', undefined, undefined)
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
+ expect(taskUtils.createTaskTemporaryDir).toHaveBeenCalledWith(user.tasksPath, 'task-d', '/data/users/john/files/archive')
+ expect(filesUtils.makeTempDir).not.toHaveBeenCalled()
+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/tasks/.task-d-archive', undefined, undefined)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
expect(taskEmitSpy).toHaveBeenCalledWith(
'startWatch',
space,
FILE_OPERATION.DECOMPRESS,
'/data/users/john/files/archive',
- '/data/users/john/tmp/archive-extract-123'
+ '/data/users/john/tmp/tasks/.task-d-archive'
)
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
})
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 9ef0fb85..be7ec99c 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -66,6 +66,7 @@ import { ACTION } from '../../../common/constants'
import { pipeline } from 'node:stream/promises'
import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'
import { FILE_ERROR_MESSAGES, maxFileSizeExceededError } from '../utils/errors'
+import { copyAbortable, createTaskTemporaryDir, moveAbortable, SourceCleanupError, taskTemporaryPath } from '../utils/tasks'
@Injectable()
export class FilesManager {
@@ -382,8 +383,10 @@ export class FilesManager {
isMove: boolean,
overwrite = false,
mkdirDstParentPath = false,
- dav?: { depth: LOCK_DEPTH; lockTokens: string[] }
+ dav?: { depth: LOCK_DEPTH; lockTokens: string[] },
+ signal?: AbortSignal
): Promise<void> {
+ const isTaskContext = Boolean(srcSpace.task?.cacheKey)
// checks
this.checkNotTrashRepository(dstSpace)
if (!canAccessToSpace(user, dstSpace)) {
@@ -472,35 +475,46 @@ export class FilesManager {
// check destination
await this.filesLockManager.checkConflicts(dstSpace.dbFile, depth, { userId: user.id, lockTokens: dav?.lockTokens })
- // overwrite
- if (overwrite && (await isPathExists(dstSpace.realPath))) {
- // todo : versioning here
+ // Task transfers defer overwrite handling until their staged content is ready to commit.
+ if (!isTaskContext && overwrite && (await isPathExists(dstSpace.realPath))) {
await this.delete(user, dstSpace)
}
- // send it to task watcher
- if (srcSpace.task?.cacheKey) {
- if (!isDir) srcSpace.task.props.totalSize = await fileSize(srcSpace.realPath)
- FileTaskEvent.emit('startWatch', srcSpace, isMove ? FILE_OPERATION.MOVE : FILE_OPERATION.COPY, dstSpace.realPath)
- }
-
// do
if (isMove) {
- await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
+ let sourceCleanupError: SourceCleanupError | undefined
+ if (isTaskContext) {
+ sourceCleanupError = await this.moveAsTask(user, srcSpace, dstSpace, overwrite, isDir, signal)
+ } else {
+ await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
+ }
// emit a file event when the source space is different from the destination space
if (srcSpace.realBasePath !== dstSpace.realBasePath) {
FileEvent.emit('event', { user, space: srcSpace, action: ACTION.DELETE_PERMANENTLY, rPath: srcSpace.realPath })
FileEvent.emit('event', { user, space: dstSpace, action: ACTION.ADD, rPath: dstSpace.realPath })
}
+ // A published cross-device destination is authoritative even if the obsolete source could not be removed.
await this.filesQueries.moveFiles(srcSpace.dbFile, dstSpace.dbFile, isDir)
+ if (sourceCleanupError) {
+ this.logSourceCleanupError(sourceCleanupError)
+ throw sourceCleanupError
+ }
} else {
- await copyFiles(srcSpace.realPath, dstSpace.realPath, overwrite, recursive)
+ if (isTaskContext) {
+ if (!signal) {
+ throw new Error('An abort signal is required for a copy task')
+ }
+ await this.copyAsTask(user, srcSpace, dstSpace, overwrite, recursive, isDir, signal)
+ } else {
+ await copyFiles(srcSpace.realPath, dstSpace.realPath, overwrite, recursive)
+ }
// emit file event
FileEvent.emit('event', { user, space: dstSpace, action: ACTION.ADD, rPath: dstSpace.realPath })
}
}
- async delete(user: UserModel, space: SpaceEnv, dav?: { lockTokens: string[] }): Promise<void> {
+ async delete(user: UserModel, space: SpaceEnv, dav?: { lockTokens: string[] }, signal?: AbortSignal): Promise<void> {
+ const isTaskContext = Boolean(space.task?.cacheKey)
if (!(await isPathExists(space.realPath))) {
throw new FileError(HttpStatus.NOT_FOUND, 'Location not found')
}
@@ -512,6 +526,7 @@ export class FilesManager {
})
// file system deletion
let forceDeleteInDB = false
+ let sourceCleanupError: SourceCleanupError | undefined
if (space.inTrashRepository) {
await removeFiles(space.realPath)
FileEvent.emit('event', { user, space, action: ACTION.DELETE_PERMANENTLY, rPath: space.realPath })
@@ -524,17 +539,12 @@ export class FilesManager {
if (!(await isPathExists(trashDir))) {
await makeDir(trashDir, true)
}
- if (await isPathExists(trashFile)) {
- // if a resource already exists in the trash, rename it with the date
- const dstTrash = await uniqueDatedFilePath(trashFile)
- // move the resource on fs
- await moveFiles(trashFile, dstTrash.path)
- // move the resource in db
- const trashFileDB: FileDBProps = { ...space.dbFile, inTrash: true }
- const dstTrashFileDB: FileDBProps = { ...trashFileDB, path: path.join(dirName(trashFileDB.path), fileName(dstTrash.path)) }
- await this.filesQueries.moveFiles(trashFileDB, dstTrashFileDB, dstTrash.isDir)
+ if (isTaskContext) {
+ sourceCleanupError = await this.deleteAsTask(user, space, trashFile, isDir, signal)
+ } else {
+ await this.moveExistingTrashFile(space, trashFile)
+ await moveFiles(space.realPath, trashFile, true)
}
- await moveFiles(space.realPath, trashFile, true)
// emit file event
if (space.dbFile.shareExternalId) {
// deleted files from shares with external locations are moved to the owner’s trash
@@ -563,15 +573,22 @@ export class FilesManager {
for (const lock of await this.filesLockManager.getLocksByPath(space.dbFile)) {
this.filesLockManager.removeLock(lock.key).catch((e: Error) => this.logger.error({ tag: this.delete.name, msg: `${e}` }))
}
- // delete or move to trash the files in db
+ // Keep the database aligned with the published trash copy before reporting a residual source.
await this.filesQueries.deleteFiles(space.dbFile, isDir, forceDeleteInDB)
+ if (sourceCleanupError) {
+ this.logSourceCleanupError(sourceCleanupError)
+ throw sourceCleanupError
+ }
}
async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto, signal?: AbortSignal): Promise<void> {
+ const isTaskContext = Boolean(space.task?.cacheKey)
this.checkNotTrashRepository(space)
this.logger.log({ tag: this.downloadFromUrl.name, msg: `${downloadDto.url}` })
const dstPath = await uniqueFilePathFromDir(space.realPath)
- const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)
+ const tmpPath = isTaskContext
+ ? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)
+ : tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)
const dbFile = space.dbFile
dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))
@@ -598,6 +615,7 @@ export class FilesManager {
}
async compress(user: UserModel, space: SpaceEnv, dto: CompressFileDto, signal?: AbortSignal): Promise<void> {
+ const isTaskContext = Boolean(space.task?.cacheKey)
// This method is currently used only by files-methods.service, which handles input sanitization.
// If it is used in other services in the future, make sure to refactor accordingly to sanitize inputs properly.
if (dto.compressInDirectory) {
@@ -606,7 +624,9 @@ export class FilesManager {
const srcPath = dirName(space.realPath)
const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`
const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))
- const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)
+ const tmpPath = isTaskContext
+ ? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)
+ : tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)
// avoid using ZIP here because it can trigger high memory usage.
const archive: Archiver = archiver(TAR_EXTENSION, {
gzip: dto.extension === TAR_GZ_EXTENSION,
@@ -625,8 +645,8 @@ export class FilesManager {
}
fileLock = lock
}
- if (space.task?.cacheKey) {
- space.task.props.compressInDirectory = dto.compressInDirectory
+ if (isTaskContext) {
+ space.task!.props.compressInDirectory = dto.compressInDirectory
FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)
}
// do
@@ -672,6 +692,7 @@ export class FilesManager {
}
async decompress(user: UserModel, space: SpaceEnv, signal?: AbortSignal): Promise<void> {
+ const isTaskContext = Boolean(space.task?.cacheKey)
// checks
this.checkNotTrashRepository(space)
if (!(await isPathExists(space.realPath))) {
@@ -683,7 +704,9 @@ export class FilesManager {
}
// make temporary extraction folder
const dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))
- const tmpPath = await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)
+ const tmpPath = isTaskContext
+ ? await createTaskTemporaryDir(user.tasksPath, space.task!.cacheKey, dstPath)
+ : await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)
let fileLock: FileLock | undefined
try {
// create lock
@@ -695,7 +718,7 @@ export class FilesManager {
}
fileLock = lock
// tasking
- if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
+ if (isTaskContext) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
// do
const maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)
if (extension === '.zip') {
@@ -798,6 +821,118 @@ export class FilesManager {
}
}
+ private async copyAsTask(
+ user: UserModel,
+ srcSpace: SpaceEnv,
+ dstSpace: SpaceEnv,
+ overwrite: boolean,
+ recursive: boolean,
+ isDir: boolean,
+ signal: AbortSignal
+ ): Promise<void> {
+ await this.setTaskTotalSize(srcSpace, isDir)
+ await copyAbortable(srcSpace.realPath, dstSpace.realPath, {
+ beforeCommit: this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite),
+ cacheKey: srcSpace.task!.cacheKey,
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),
+ overwrite,
+ recursive,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ private async moveAsTask(
+ user: UserModel,
+ srcSpace: SpaceEnv,
+ dstSpace: SpaceEnv,
+ overwrite: boolean,
+ isDir: boolean,
+ signal?: AbortSignal
+ ): Promise<SourceCleanupError | undefined> {
+ await this.setTaskTotalSize(srcSpace, isDir)
+ const beforeCommit = this.prepareTaskDestination(user, srcSpace, dstSpace, overwrite)
+ if (!signal) {
+ // Same-device moves stay on the regular atomic path and are therefore not cancellable.
+ this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)
+ await beforeCommit?.()
+ await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
+ return
+ }
+ return moveAbortable(srcSpace.realPath, dstSpace.realPath, {
+ beforeCommit,
+ cacheKey: srcSpace.task!.cacheKey,
+ crossDevice: true,
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),
+ overwrite,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ private async deleteAsTask(
+ user: UserModel,
+ space: SpaceEnv,
+ trashFile: string,
+ isDir: boolean,
+ signal?: AbortSignal
+ ): Promise<SourceCleanupError | undefined> {
+ await this.setTaskTotalSize(space, isDir)
+ const beforeCommit = () => this.moveExistingTrashFile(space, trashFile)
+ if (!signal) {
+ // Moving to trash on the same device stays atomic; only cross-device deletes receive a signal.
+ this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)
+ await beforeCommit()
+ await moveFiles(space.realPath, trashFile, true)
+ return
+ }
+ return moveAbortable(space.realPath, trashFile, {
+ beforeCommit,
+ cacheKey: space.task!.cacheKey,
+ crossDevice: true,
+ onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),
+ overwrite: true,
+ signal,
+ stagingDir: user.tasksPath
+ })
+ }
+
+ private prepareTaskDestination(user: UserModel, srcSpace: SpaceEnv, dstSpace: SpaceEnv, overwrite: boolean): (() => Promise<void>) | undefined {
+ if (!overwrite || srcSpace.realPath.toLowerCase() === dstSpace.realPath.toLowerCase()) return
+ // Preserve the current destination until the staged transfer reaches its commit phase.
+ return async () => {
+ if (await isPathExists(dstSpace.realPath)) {
+ await this.delete(user, dstSpace)
+ }
+ }
+ }
+
+ private async moveExistingTrashFile(space: SpaceEnv, trashFile: string): Promise<void> {
+ if (!(await isPathExists(trashFile))) return
+ const dstTrash = await uniqueDatedFilePath(trashFile)
+ await moveFiles(trashFile, dstTrash.path)
+ const trashFileDB: FileDBProps = { ...space.dbFile, inTrash: true }
+ const dstTrashFileDB: FileDBProps = { ...trashFileDB, path: path.join(dirName(trashFileDB.path), fileName(dstTrash.path)) }
+ await this.filesQueries.moveFiles(trashFileDB, dstTrashFileDB, dstTrash.isDir)
+ }
+
+ private async setTaskTotalSize(space: SpaceEnv, isDir: boolean): Promise<void> {
+ if (!isDir) {
+ space.task!.props.totalSize = await fileSize(space.realPath)
+ }
+ }
+
+ private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {
+ FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)
+ }
+
+ private logSourceCleanupError(error: SourceCleanupError): void {
+ this.logger.error({
+ tag: this.logSourceCleanupError.name,
+ msg: `${error.message}: ${error.srcPath} -> ${error.dstPath}: ${error.cause}`
+ })
+ }
+
private checkNotTrashRepository(space: SpaceEnv): void {
if (space.inTrashRepository) {
throw new FileError(HttpStatus.FORBIDDEN, 'The trash is read-only')
diff --git a/backend/src/applications/files/services/files-methods.service.spec.ts b/backend/src/applications/files/services/files-methods.service.spec.ts
index db96b5e8..32daefee 100644
--- a/backend/src/applications/files/services/files-methods.service.spec.ts
+++ b/backend/src/applications/files/services/files-methods.service.spec.ts
@@ -1,5 +1,7 @@
+import { HttpException, HttpStatus } from '@nestjs/common'
import { Test, TestingModule } from '@nestjs/testing'
import path from 'node:path'
+import type { Mock } from 'vitest'
import { transformAndValidate } from '../../../common/functions'
import { Cache } from '../../../infrastructure/cache/cache.service'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
@@ -24,6 +26,7 @@ describe(FilesMethods.name, () => {
let filesMethods: FilesMethods
let spacesManager: SpacesManager
let filesQuotaManager: FilesQuotaManager
+ let filesManager: { compress: Mock; delete: Mock }
let userTest: UserModel
const spaceEnv = {
id: 1,
@@ -38,6 +41,10 @@ describe(FilesMethods.name, () => {
} as SpaceEnv
beforeAll(async () => {
+ filesManager = {
+ compress: vi.fn(),
+ delete: vi.fn()
+ }
const module: TestingModule = await Test.createTestingModule({
imports: [],
providers: [
@@ -64,9 +71,7 @@ describe(FilesMethods.name, () => {
},
{
provide: FilesManager,
- useValue: {
- compress: vi.fn()
- }
+ useValue: filesManager
}
]
}).compile()
@@ -104,4 +109,28 @@ describe(FilesMethods.name, () => {
compressFileDto.files[0].path = '../../../bar/../'
await expect(filesMethods.compress(userTest, spaceEnv, compressFileDto)).rejects.toThrow(/is not valid/i)
})
+
+ it('should preserve the cancellation reason from an abort error', async () => {
+ const controller = new AbortController()
+ const reason = new Error('Cancelled')
+ const abortError = Object.assign(new Error('The operation was aborted', { cause: reason }), {
+ code: 'ABORT_ERR',
+ name: 'AbortError'
+ })
+ controller.abort(reason)
+ filesManager.delete.mockRejectedValueOnce(abortError)
+
+ await expect(filesMethods.delete(userTest, spaceEnv, null, controller.signal)).rejects.toBe(reason)
+ })
+
+ it('should keep a late operation error after the signal was aborted', async () => {
+ const controller = new AbortController()
+ const operationError = new Error('source cleanup failed')
+ controller.abort(new Error('Cancelled'))
+ filesManager.delete.mockRejectedValueOnce(operationError)
+
+ await expect(filesMethods.delete(userTest, spaceEnv, null, controller.signal)).rejects.toEqual(
+ new HttpException(operationError.message, HttpStatus.INTERNAL_SERVER_ERROR)
+ )
+ })
})
diff --git a/backend/src/applications/files/services/files-methods.service.ts b/backend/src/applications/files/services/files-methods.service.ts
index 548f68f7..ffd3d8fe 100644
--- a/backend/src/applications/files/services/files-methods.service.ts
+++ b/backend/src/applications/files/services/files-methods.service.ts
@@ -55,19 +55,19 @@ export class FilesMethods {
}
}
- copy(user: UserModel, space: SpaceEnv, copyMoveFileDto: CopyMoveFileDto): Promise<CopyMoveFileResponse> {
- return this.copyMove(user, space, copyMoveFileDto, false)
+ copy(user: UserModel, space: SpaceEnv, copyMoveFileDto: CopyMoveFileDto, signal?: AbortSignal): Promise<CopyMoveFileResponse> {
+ return this.copyMove(user, space, copyMoveFileDto, false, signal)
}
- move(user: UserModel, space: SpaceEnv, copyMoveFileDto: CopyMoveFileDto): Promise<CopyMoveFileResponse> {
- return this.copyMove(user, space, copyMoveFileDto, true)
+ move(user: UserModel, space: SpaceEnv, copyMoveFileDto: CopyMoveFileDto, signal?: AbortSignal): Promise<CopyMoveFileResponse> {
+ return this.copyMove(user, space, copyMoveFileDto, true, signal)
}
- async delete(user: UserModel, space: SpaceEnv): Promise<void> {
+ async delete(user: UserModel, space: SpaceEnv, _dto?: null, signal?: AbortSignal): Promise<void> {
try {
- return await this.filesManager.delete(user, space)
+ return await this.filesManager.delete(user, space, undefined, signal)
} catch (e) {
- this.handleError(space, FILE_OPERATION.DELETE, e)
+ this.handleError(space, FILE_OPERATION.DELETE, e, undefined, signal)
}
}
@@ -76,7 +76,7 @@ export class FilesMethods {
try {
return await this.filesManager.downloadFromUrl(user, space, downloadDto, signal)
} catch (e) {
- this.handleError(space, FILE_OPERATION.DOWNLOAD, e)
+ this.handleError(space, FILE_OPERATION.DOWNLOAD, e, undefined, signal)
}
}
@@ -109,7 +109,7 @@ export class FilesMethods {
}
return await this.filesManager.compress(user, space, compressFileDto, signal)
} catch (e) {
- this.handleError(space, FILE_OPERATION.COMPRESS, e)
+ this.handleError(space, FILE_OPERATION.COMPRESS, e, undefined, signal)
}
}
@@ -117,7 +117,7 @@ export class FilesMethods {
try {
return await this.filesManager.decompress(user, space, signal)
} catch (e) {
- this.handleError(space, FILE_OPERATION.DECOMPRESS, e)
+ this.handleError(space, FILE_OPERATION.DECOMPRESS, e, undefined, signal)
}
}
@@ -165,19 +165,28 @@ export class FilesMethods {
}
}
- private async copyMove(user: UserModel, space: SpaceEnv, copyMoveFileDto: CopyMoveFileDto, isMove: boolean): Promise<CopyMoveFileResponse> {
- const dstUrl = path.join(copyMoveFileDto.dstDirectory, copyMoveFileDto.dstName ? copyMoveFileDto.dstName : fileName(space.realPath))
+ private async copyMove(
+ user: UserModel,
+ space: SpaceEnv,
+ copyMoveFileDto: CopyMoveFileDto,
+ isMove: boolean,
+ signal?: AbortSignal
+ ): Promise<CopyMoveFileResponse> {
+ const dstUrl = path.join(copyMoveFileDto.dstDirectory, copyMoveFileDto.dstName || fileName(space.realPath))
let dstSpace: SpaceEnv
try {
dstSpace = await this.spacesManager.spaceEnv(user, dstUrl.split('/'))
- await this.filesManager.copyMove(user, space, dstSpace, isMove, copyMoveFileDto.overwrite)
+ await this.filesManager.copyMove(user, space, dstSpace, isMove, copyMoveFileDto.overwrite, false, undefined, signal)
return { path: dirName(dstUrl), name: fileName(dstUrl), mime: getMimeType(dstSpace.realPath, await isPathIsDir(dstSpace.realPath)) }
} catch (e) {
- this.handleError(space, isMove ? FILE_OPERATION.MOVE : FILE_OPERATION.COPY, e, dstSpace)
+ this.handleError(space, isMove ? FILE_OPERATION.MOVE : FILE_OPERATION.COPY, e, dstSpace, signal)
}
}
- private handleError(space: SpaceEnv, action: string, e: any, dstSpace?: SpaceEnv): never {
+ private handleError(space: SpaceEnv, action: string, e: any, dstSpace?: SpaceEnv, signal?: AbortSignal): never {
+ if (signal && this.isCancellationError(e, signal)) {
+ throw signal.reason
+ }
this.logger.error({ tag: this.handleError.name, msg: `unable to ${action} ${space.url}${dstSpace?.url ? ` -> ${dstSpace.url}` : ''} : ${e}` })
// Remove the last part to avoid exposing the path
const errorMsg = e.message.split(',')[0]
@@ -188,4 +197,12 @@ export class FilesMethods {
}
throw new HttpException(errorMsg, HttpStatus.INTERNAL_SERVER_ERROR)
}
+
+ private isCancellationError(error: unknown, signal?: AbortSignal): boolean {
+ if (!signal?.aborted) return false
+ if (error === signal.reason) return true
+ if (!(error instanceof Error)) return false
+ const code = (error as NodeJS.ErrnoException).code
+ return error.cause === signal.reason || error.name === 'AbortError' || code === 'ABORT_ERR' || code === 'ERR_CANCELED'
+ }
}
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 44b10dc8..9924f45a 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -17,6 +17,7 @@ import { FileTask, FileTaskStatus } from '../models/file-task'
import { filesRecents } from '../schemas/files-recents.schema'
import { files } from '../schemas/files.schema'
import { isPathExists, removeFiles } from '../utils/files'
+import { taskTemporaryPrefix } from '../utils/tasks'
import { FilesContentIndexer } from './files-content-indexer.service'
import { FilesTasksManager } from './files-tasks-manager.service'
import { FilesQuotaManager } from './files-quota-manager.service'
@@ -212,19 +213,26 @@ export class FilesScheduler {
this.isQuotaUpdateIsRunning = false
}
- private async cleanupUserTaskFiles(userId: number, userTasksPath: string): Promise<void> {
+ private async cleanupUserTaskFiles(userId: number, userTasksPath: string, expiration: number): Promise<void> {
try {
const keys = await this.cache.keys(FilesTasksManager.getCacheKey(userId))
- const excludeFiles = keys.length
- ? (await this.cache.mget(keys))
- .filter((task: FileTask) => task && task.status === FileTaskStatus.PENDING && task.props.compressInDirectory === false)
- .map((task: FileTask) => task.name)
- : []
- for (const f of (await fs.readdir(userTasksPath)).filter((f: string) => excludeFiles.indexOf(f) === -1)) {
- await this.removeTmpFile(path.join(userTasksPath, f))
+ const tasks: (FileTask | null | undefined)[] = keys.length ? await this.cache.mget(keys) : []
+ const protectedFiles = new Set<string>()
+ const protectedPrefixes: string[] = []
+ for (const task of tasks) {
+ if (!task || !this.isActiveTaskStatus(task.status)) continue
+ // QUEUED tasks are included because they may start after this cache snapshot.
+ protectedPrefixes.push(taskTemporaryPrefix(FilesTasksManager.getCacheKey(userId, task.id)))
+ // Exported archives are final results stored beside staging entries and do not use the task prefix.
+ if (task.props.compressInDirectory === false) protectedFiles.add(task.name)
+ }
+ for (const fileName of await fs.readdir(userTasksPath)) {
+ if (protectedFiles.has(fileName) || protectedPrefixes.some((prefix: string) => fileName.startsWith(prefix))) continue
+ // The age guard protects tasks created after the cache snapshot from being removed by this cleanup pass.
+ await this.removeTmpFile(path.join(userTasksPath, fileName), expiration)
}
} catch (e) {
- this.logger.error({ tag: this.cleanupUserTmpFiles.name, msg: `unable to browse ${userTasksPath} : ${e}` })
+ this.logger.error({ tag: this.cleanupUserTaskFiles.name, msg: `unable to browse ${userTasksPath} : ${e}` })
}
}
@@ -237,7 +245,7 @@ export class FilesScheduler {
for (const f of await fs.readdir(userTmpPath)) {
const rPath = path.join(userTmpPath, f)
if (f === USER_PATH.TASKS) {
- await this.cleanupUserTaskFiles(user.id, rPath)
+ await this.cleanupUserTaskFiles(user.id, rPath, expiration)
} else {
await this.removeTmpFile(rPath, expiration)
}
@@ -275,7 +283,7 @@ export class FilesScheduler {
continue
}
const task = await this.cache.get(key)
- if (task && this.isInterruptedTaskStatus(task.status)) {
+ if (task && this.isActiveTaskStatus(task.status)) {
task.status = FileTaskStatus.ERROR
task.result = 'Interrupted'
task.endedAt = currentTimeStamp(null, true)
@@ -292,7 +300,7 @@ export class FilesScheduler {
}
}
- private isInterruptedTaskStatus(status: FileTaskStatus): boolean {
+ private isActiveTaskStatus(status: FileTaskStatus): boolean {
return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
}
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
index 09b2e9fb..9b9c9037 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
@@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing'
import { HttpException, HttpStatus } from '@nestjs/common'
import crypto from 'node:crypto'
import { Cache } from '../../../infrastructure/cache/cache.service'
+import { SpacesManager } from '../../spaces/services/spaces-manager.service'
import * as filesUtils from '../utils/files'
import { FileTaskEvent } from '../events/file-events'
import { SendFile } from '../utils/send-file'
@@ -19,6 +20,7 @@ describe(FilesTasksManager.name, () => {
let filesMethods: {
doWork: Mock
}
+ let spacesManager: { spaceEnv: Mock }
let cacheStore: Map<string, any>
let cache: {
set: Mock
@@ -39,6 +41,30 @@ describe(FilesTasksManager.name, () => {
return new RegExp(`^${escapedPattern}$`)
}
+ const runCancelledTask = async (taskId: `${string}-${string}-${string}-${string}-${string}`, rejection?: Error) => {
+ const user = { id: 7 } as any
+ const space = { url: 'files/personal/document.txt' } as any
+ const cacheKey = `${CACHE_TASK_PREFIX}-7-${taskId}`
+ const cancellationCacheKey = `${CACHE_TASK_CANCEL_PREFIX}-7-${taskId}`
+ let signal!: AbortSignal
+
+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)
+ filesMethods.doWork.mockImplementationOnce((_user, _space, _dto, taskSignal?: AbortSignal) => {
+ if (!taskSignal) return Promise.reject(new Error('missing abort signal'))
+ signal = taskSignal
+ return new Promise((_resolve, reject) => {
+ taskSignal.addEventListener('abort', () => reject(rejection ?? taskSignal.reason), { once: true })
+ })
+ })
+
+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, space, null, 'doWork')
+ cacheStore.set(cancellationCacheKey, true)
+ await vi.advanceTimersByTimeAsync(1000)
+ for (let i = 0; i < 6; i++) await Promise.resolve()
+
+ return { cancellationCacheKey, signal, storedTask: cacheStore.get(cacheKey) }
+ }
+
beforeEach(async () => {
cacheStore = new Map<string, any>()
cache = {
@@ -56,8 +82,9 @@ describe(FilesTasksManager.name, () => {
}),
del: vi.fn(async (key: string) => cacheStore.delete(key))
}
- filesMethods = {
- doWork: vi.fn()
+ filesMethods = { doWork: vi.fn() }
+ spacesManager = {
+ spaceEnv: vi.fn().mockResolvedValue({ realPath: '/data/users/john/files/destination.txt' })
}
module = await Test.createTestingModule({
providers: [
@@ -67,6 +94,7 @@ describe(FilesTasksManager.name, () => {
provide: Cache,
useValue: cache
},
+ { provide: SpacesManager, useValue: spacesManager },
{ provide: FilesMethods, useValue: filesMethods }
]
}).compile()
@@ -101,6 +129,31 @@ describe(FilesTasksManager.name, () => {
expect(FileTaskEvent.listenerCount('startWatch')).toBe(listenersBeforeDestroy - 1)
})
+ it('should not overlap watcher updates', async () => {
+ vi.useFakeTimers()
+ try {
+ let finishUpdate: () => void
+ const update = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ finishUpdate = resolve
+ })
+ )
+ ;(filesTasksManager as any).watchTask('task-1', update)
+
+ await vi.advanceTimersByTimeAsync(2000)
+ expect(update).toHaveBeenCalledTimes(1)
+
+ finishUpdate()
+ await Promise.resolve()
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(update).toHaveBeenCalledTimes(2)
+ ;(filesTasksManager as any).stopWatch('task-1')
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('should build task cache key with and without task id', () => {
expect(FilesTasksManager.getCacheKey(10, 'task-1')).toBe(`${CACHE_TASK_PREFIX}-10-task-1`)
expect(FilesTasksManager.getCacheKey(10)).toBe(`${CACHE_TASK_PREFIX}-10-*`)
@@ -137,34 +190,10 @@ describe(FilesTasksManager.name, () => {
it('should abort a cancellable task and store a cancelled status when cancellation is requested', async () => {
vi.useFakeTimers()
try {
- const taskId = '33333333-3333-4333-8333-333333333333'
- const user = { id: 7 } as any
- const space = { url: 'files/personal/document.txt' } as any
- const cacheKey = `${CACHE_TASK_PREFIX}-7-${taskId}`
- const cancellationCacheKey = `${CACHE_TASK_CANCEL_PREFIX}-7-${taskId}`
- let taskSignal!: AbortSignal
-
- vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)
- filesMethods.doWork.mockImplementationOnce((_user, _space, _dto, signal?: AbortSignal) => {
- if (!signal) return Promise.reject(new Error('missing abort signal'))
- taskSignal = signal
- return new Promise((_resolve, reject) => {
- signal.addEventListener('abort', () => reject(signal.reason), { once: true })
- })
- })
-
- const task = await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, space, null, 'doWork')
- expect(task.cancellable).toBe(true)
- cacheStore.set(cancellationCacheKey, true)
-
- await vi.advanceTimersByTimeAsync(1000)
- for (let i = 0; i < 6; i++) await Promise.resolve()
+ const { cancellationCacheKey, signal, storedTask } = await runCancelledTask('33333333-3333-4333-8333-333333333333')
- const storedTask = cacheStore.get(cacheKey)
- expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, null, expect.any(AbortSignal))
- expect(taskSignal.aborted).toBe(true)
- expect(storedTask.status).toBe(FileTaskStatus.CANCELLED)
- expect(storedTask.result).toBe('Cancelled')
+ expect(signal.aborted).toBe(true)
+ expect(storedTask).toMatchObject({ status: FileTaskStatus.CANCELLED, result: 'Cancelled' })
expect(cacheStore.has(cancellationCacheKey)).toBe(false)
expect((filesTasksManager as any).tasksCancellationWatcher).toEqual({})
} finally {
@@ -172,6 +201,18 @@ describe(FilesTasksManager.name, () => {
}
})
+ it('should keep a late task error when the signal was already aborted', async () => {
+ vi.useFakeTimers()
+ try {
+ const taskError = new Error('source cleanup failed')
+ const { storedTask } = await runCancelledTask('55555555-5555-4555-8555-555555555555', taskError)
+
+ expect(storedTask).toMatchObject({ status: FileTaskStatus.ERROR, result: taskError.message })
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('should watch a temporary path while keeping the published file name', () => {
const startWatchSpy = vi.spyOn(filesTasksManager as any, 'startWatch').mockResolvedValueOnce(undefined)
const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any
@@ -269,6 +310,40 @@ describe(FilesTasksManager.name, () => {
expect(stopWatchSpy).toHaveBeenCalledWith('task-1')
})
+ it('should not let an in-flight progress update overwrite task completion', async () => {
+ const cacheKey = `${CACHE_TASK_PREFIX}-7-task-pending`
+ cacheStore.set(cacheKey, {
+ id: 'task-pending',
+ status: FileTaskStatus.PENDING,
+ props: { size: 10 }
+ })
+ let releaseGet: () => void
+ let notifyGetStarted: () => void
+ const getStarted = new Promise<void>((resolve) => {
+ notifyGetStarted = resolve
+ })
+ const resumeGet = new Promise<void>((resolve) => {
+ releaseGet = resolve
+ })
+ cache.get.mockImplementationOnce(async (key: string) => {
+ notifyGetStarted()
+ await resumeGet
+ return cacheStore.get(key)
+ })
+
+ const updatePromise = (filesTasksManager as any).updateTask(cacheKey, { size: 20 })
+ await getStarted
+ const completionPromise = (filesTasksManager as any).setTaskDone(cacheKey, FileTaskStatus.SUCCESS, 'done')
+ releaseGet()
+ await Promise.all([updatePromise, completionPromise])
+
+ expect(cacheStore.get(cacheKey)).toMatchObject({
+ status: FileTaskStatus.SUCCESS,
+ result: 'done',
+ props: { size: 10 }
+ })
+ })
+
it('should create a task and mark it as success when the async method resolves', async () => {
vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
filesMethods.doWork.mockResolvedValueOnce({ props: { totalSize: 99 }, result: 'done' })
@@ -279,19 +354,17 @@ describe(FilesTasksManager.name, () => {
await flushPromises()
expect(task.id).toBe('11111111-1111-4111-8111-111111111111')
- expect(task.cancellable).toBe(false)
+ expect(task.cancellable).toBe(true)
expect(task.status).toBe(FileTaskStatus.SUCCESS)
expect(space.task.cacheKey).toBe(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`)
- expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, { foo: 'bar' })
- expect(cache.set).toHaveBeenCalledWith(
- `${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`,
- expect.objectContaining({ id: '11111111-1111-4111-8111-111111111111', status: FileTaskStatus.SUCCESS, result: 'done' }),
- CACHE_TASK_TTL
- )
+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, { foo: 'bar' }, expect.any(AbortSignal))
const storedTask = cacheStore.get(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`)
- expect(storedTask.status).toBe(FileTaskStatus.SUCCESS)
+ expect(storedTask).toMatchObject({
+ status: FileTaskStatus.SUCCESS,
+ result: 'done',
+ props: { totalSize: 99 }
+ })
expect(storedTask.endedAt).toBeDefined()
- expect(storedTask.props.totalSize).toBe(99)
})
it('should create a task and mark it as error when the async method fails', async () => {
@@ -308,6 +381,20 @@ describe(FilesTasksManager.name, () => {
expect(storedTask.result).toBe('operation failed')
})
+ it('should run a non-cancellable task without an abort signal', async () => {
+ filesMethods.doWork.mockResolvedValueOnce(undefined)
+ const user = { id: 7 } as any
+ const space = { realPath: '/data/users/john/files/document.txt', url: 'files/personal/document.txt' } as any
+ const dto = { dstDirectory: 'files/personal' }
+
+ const task = await filesTasksManager.createTask(FILE_OPERATION.MOVE, user, space, dto, 'doWork')
+ await flushPromises()
+
+ expect(spacesManager.spaceEnv).toHaveBeenCalledWith(user, ['files', 'personal', 'document.txt'])
+ expect(task.cancellable).toBe(false)
+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, dto)
+ })
+
it('should queue tasks when a user already has three running tasks', async () => {
vi.spyOn(crypto, 'randomUUID')
.mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.ts
index a11f9f7f..3b71c47e 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts
@@ -6,30 +6,38 @@ import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/
import { currentTimeStamp } from '../../../common/shared'
import { Cache } from '../../../infrastructure/cache/cache.service'
import { SpaceEnv } from '../../spaces/models/space-env.model'
+import { SpacesManager } from '../../spaces/services/spaces-manager.service'
+import { realTrashPathFromSpace } from '../../spaces/utils/paths'
import { UserModel } from '../../users/models/user.model'
import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
import { FILE_OPERATION } from '../constants/operations'
+import { CopyMoveFileDto } from '../dto/file-operations.dto'
import { FileTaskEvent } from '../events/file-events'
import type { FileTaskQueueItem } from '../interfaces/file-task-queue.interface'
import type { FileTasksPollResponse } from '../interfaces/file-task.interface'
import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'
import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'
import { SendFile } from '../utils/send-file'
+import { isTaskCancellable } from '../utils/tasks'
import { FilesMethods } from './files-methods.service'
import { FilesTasksQueue } from './files-tasks-queue.service'
@Injectable()
export class FilesTasksManager implements OnModuleDestroy {
- // task cache key = `ftask-$(userId}-${taskId}` => FileTask
+ // Task cache key: `ftask-${userId}-${taskId}`.
private readonly logger = new Logger(FilesTasksManager.name)
private readonly watchInterval = 1000
private tasksWatcher: Record<string, NodeJS.Timeout> = {}
private tasksCancellationWatcher: Record<string, NodeJS.Timeout> = {}
+ // Cache writes are serialized per task, while finalization blocks any late progress write.
+ private pendingTaskUpdates = new Map<string, Promise<void>>()
+ private finalizingTasks = new Set<string>()
constructor(
private readonly cache: Cache,
private readonly filesMethods: FilesMethods,
- private readonly filesTasksQueue: FilesTasksQueue
+ private readonly filesTasksQueue: FilesTasksQueue,
+ private readonly spacesManager: SpacesManager
) {
FileTaskEvent.on('startWatch', this.onStartWatch)
}
@@ -55,7 +63,10 @@ export class FilesTasksManager implements OnModuleDestroy {
async createTask(type: FILE_OPERATION, user: UserModel, space: SpaceEnv, dto: any, method: string): Promise<FileTask> {
const taskId: string = crypto.randomUUID()
const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)
- const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url), this.isCancellableType(type))
+ // MOVE and DELETE need their effective destination to determine whether they cross devices.
+ const dstPath = await this.taskDestinationPath(type, user, space, dto)
+ const cancellable = await isTaskCancellable(type, space.realPath, dstPath)
+ const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url), cancellable)
await this.storeTask(cacheKey, newTask, FileTaskStatus.QUEUED)
space.task = { cacheKey, props: {} }
await this.filesTasksQueue.enqueue(user.id, { cacheKey, dto, method, space, task: newTask, user }, (task) => this.startQueuedTask(task))
@@ -157,24 +168,30 @@ export class FilesTasksManager implements OnModuleDestroy {
}
private async setTaskDone(cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {
- const task: FileTask = await this.cache.get(cacheKey)
- if (task) {
- task.status = status
- task.endedAt = currentTimeStamp(null, true)
- if (result) {
- if (typeof result === 'string') {
- task.result = result
- } else {
- Object.assign(task, result)
+ this.finalizingTasks.add(cacheKey)
+ try {
+ this.stopWatch(cacheKey)
+ this.stopCancellationWatch(cacheKey)
+ await this.pendingTaskUpdates.get(cacheKey)?.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
+ const task: FileTask = await this.cache.get(cacheKey)
+ if (task) {
+ task.status = status
+ task.endedAt = currentTimeStamp(null, true)
+ if (result) {
+ if (typeof result === 'string') {
+ task.result = result
+ } else {
+ Object.assign(task, result)
+ }
}
+ await this.cache.set(cacheKey, task, CACHE_TASK_TTL)
}
- await this.cache.set(cacheKey, task, CACHE_TASK_TTL)
+ await this.cache
+ .del(cacheKey.replace(`${CACHE_TASK_PREFIX}-`, `${CACHE_TASK_CANCEL_PREFIX}-`))
+ .catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
+ } finally {
+ this.finalizingTasks.delete(cacheKey)
}
- this.stopWatch(cacheKey)
- this.stopCancellationWatch(cacheKey)
- await this.cache
- .del(cacheKey.replace(`${CACHE_TASK_PREFIX}-`, `${CACHE_TASK_CANCEL_PREFIX}-`))
- .catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
}
private async startQueuedTask(task: FileTaskQueueItem): Promise<boolean> {
@@ -189,7 +206,8 @@ export class FilesTasksManager implements OnModuleDestroy {
}
private runTask(task: FileTaskQueueItem): void {
- const controller = this.watchCancellation(task.task.type, task.user.id, task.task.id, task.cacheKey)
+ // Only cancellable tasks receive a signal; its absence keeps same-device moves on the atomic path.
+ const controller = this.watchCancellation(task.task, task.user.id, task.cacheKey)
const taskPromise = controller
? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)
: this.filesMethods[task.method](task.user, task.space, task.dto)
@@ -202,11 +220,12 @@ export class FilesTasksManager implements OnModuleDestroy {
})
.catch((e: HttpException | any) => {
this.logger.warn({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} : ${e}` })
+ const isCancellation = this.isCancellationError(e, controller)
this.completeTask(
task.user.id,
task.cacheKey,
- controller?.signal.aborted ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,
- controller?.signal.aborted ? 'Cancelled' : e.message
+ isCancellation ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,
+ isCancellation ? 'Cancelled' : e.message
).catch((e: Error) => this.logger.error({ tag: this.runTask.name, msg: `${e}` }))
})
}
@@ -223,17 +242,56 @@ export class FilesTasksManager implements OnModuleDestroy {
return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
}
- private async updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {
- let ftask: FileTask = await this.cache.get(cacheKey)
- if (ftask) {
- if (task) ftask = { ...ftask, ...task }
- if (props) ftask.props = { ...ftask.props, ...props }
- await this.cache.set(cacheKey, ftask, CACHE_TASK_TTL)
- } else {
- this.stopWatch(cacheKey)
+ private isCancellationError(error: unknown, controller?: AbortController): boolean {
+ return controller?.signal.aborted === true && error === controller.signal.reason
+ }
+
+ private async taskDestinationPath(type: FILE_OPERATION, user: UserModel, space: SpaceEnv, dto: unknown): Promise<string | undefined> {
+ try {
+ if (type === FILE_OPERATION.MOVE) {
+ const copyMoveDto = dto as CopyMoveFileDto
+ const dstUrl = path.join(copyMoveDto.dstDirectory, copyMoveDto.dstName || fileName(space.realPath))
+ return (await this.spacesManager.spaceEnv(user, dstUrl.split('/'))).realPath
+ }
+ if (type === FILE_OPERATION.DELETE && !space.inTrashRepository) {
+ const baseTrashPath = realTrashPathFromSpace(user, space)
+ if (baseTrashPath) {
+ return path.join(baseTrashPath, dirName(space.dbFile.path), fileName(space.realPath))
+ }
+ }
+ } catch {
+ // Capability detection is best-effort; execution performs the authoritative validation later.
+ return
}
}
+ private updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {
+ if (this.finalizingTasks.has(cacheKey)) return Promise.resolve()
+ const previousUpdate = this.pendingTaskUpdates.get(cacheKey) ?? Promise.resolve()
+ const update = previousUpdate
+ .catch(() => undefined)
+ .then(async () => {
+ if (this.finalizingTasks.has(cacheKey)) return
+ let ftask: FileTask = await this.cache.get(cacheKey)
+ if (!ftask) {
+ this.stopWatch(cacheKey)
+ return
+ }
+ // Finalization may have started while the cache read was pending.
+ if (!this.isActiveStatus(ftask.status) || this.finalizingTasks.has(cacheKey)) {
+ this.stopWatch(cacheKey)
+ return
+ }
+ if (task) ftask = { ...ftask, ...task }
+ if (props) ftask.props = { ...ftask.props, ...props }
+ await this.cache.set(cacheKey, ftask, CACHE_TASK_TTL)
+ })
+ this.pendingTaskUpdates.set(cacheKey, update)
+ return update.finally(() => {
+ if (this.pendingTaskUpdates.get(cacheKey) === update) this.pendingTaskUpdates.delete(cacheKey)
+ })
+ }
+
private startWatch(
space: SpaceEnv,
taskType: FILE_OPERATION,
@@ -242,25 +300,28 @@ export class FilesTasksManager implements OnModuleDestroy {
taskName = fileName(rPath),
publishedPath?: string
): void {
- if (!space.task?.cacheKey || space.task.cacheKey in this.tasksWatcher) return
- this.logger.verbose({ tag: this.startWatch.name, msg: `${space.task.cacheKey}` })
- this.updateTask(space.task.cacheKey, space.task?.props, {
+ const taskContext = space.task
+ if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return
+ const { cacheKey, props } = taskContext
+ this.logger.verbose({ tag: this.startWatch.name, msg: cacheKey })
+ this.updateTask(cacheKey, props, {
name: taskName,
path: taskPath
}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
switch (taskType) {
case FILE_OPERATION.COMPRESS:
- this.watchTask(space.task.cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))
+ this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))
return
case FILE_OPERATION.DECOMPRESS:
- this.watchTask(space.task.cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))
+ this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))
return
case FILE_OPERATION.DOWNLOAD:
- this.watchTask(space.task.cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))
+ this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))
return
case FILE_OPERATION.COPY:
+ case FILE_OPERATION.DELETE:
case FILE_OPERATION.MOVE:
- this.watchTask(space.task.cacheKey, () => this.updateCopyMoveTask(space, rPath))
+ this.watchTask(cacheKey, () => this.updateCopyMoveTask(space, rPath))
return
default:
this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })
@@ -279,15 +340,26 @@ export class FilesTasksManager implements OnModuleDestroy {
}
private watchTask(cacheKey: string, update: () => Promise<void>): void {
- const watcher = setInterval(() => void update(), this.watchInterval)
+ // Directory scans may exceed the interval, so ticks use exhaust semantics.
+ let updateInProgress = false
+ const watcher = setInterval(() => {
+ if (updateInProgress) return
+ updateInProgress = true
+ void update()
+ .catch((e: Error) => this.logger.error({ tag: this.watchTask.name, msg: `${e}` }))
+ .finally(() => {
+ updateInProgress = false
+ })
+ }, this.watchInterval)
watcher.unref()
this.tasksWatcher[cacheKey] = watcher
}
- private watchCancellation(type: FILE_OPERATION, userId: number, taskId: string, cacheKey: string): AbortController | undefined {
- if (!this.isCancellableType(type)) return
+ private watchCancellation(task: FileTask, userId: number, cacheKey: string): AbortController | undefined {
+ if (!task.cancellable) return
const controller = new AbortController()
- const cancellationCacheKey = FilesTasksManager.getCancellationCacheKey(userId, taskId)
+ const cancellationCacheKey = FilesTasksManager.getCancellationCacheKey(userId, task.id)
+ // Cancellation is requested independently through the shared task cache.
const watcher = setInterval(() => void this.abortTaskIfRequested(cacheKey, cancellationCacheKey, controller), this.watchInterval)
watcher.unref()
this.tasksCancellationWatcher[cacheKey] = watcher
@@ -311,18 +383,12 @@ export class FilesTasksManager implements OnModuleDestroy {
delete this.tasksCancellationWatcher[cacheKey]
}
- private isCancellableType(type: FILE_OPERATION): boolean {
- return type === FILE_OPERATION.DOWNLOAD || type === FILE_OPERATION.COMPRESS || type === FILE_OPERATION.DECOMPRESS
- }
-
private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
try {
const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
if (size === undefined) return
space.task.props.size = size
- await this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
- this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
- )
+ await this.updateTask(space.task.cacheKey, space.task.props)
} catch (e) {
this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
this.stopWatch(space.task.cacheKey)
@@ -334,9 +400,7 @@ export class FilesTasksManager implements OnModuleDestroy {
const props = await this.readWatchedPath(rPath, publishedPath, countDirEntries)
if (props === undefined) return
space.task.props = props
- this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
- this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
- )
+ await this.updateTask(space.task.cacheKey, space.task.props)
} catch (e) {
this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
this.stopWatch(space.task.cacheKey)
@@ -346,9 +410,7 @@ export class FilesTasksManager implements OnModuleDestroy {
private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
try {
if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return
- this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
- this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
- )
+ await this.updateTask(space.task.cacheKey, space.task.props)
} catch (e) {
this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
this.stopWatch(space.task.cacheKey)
@@ -362,9 +424,7 @@ export class FilesTasksManager implements OnModuleDestroy {
} else {
await this.calcSizeAndProgressTask(space, rPath)
}
- this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
- this.logger.error({ tag: this.updateCopyMoveTask.name, msg: `${e}` })
- )
+ await this.updateTask(space.task.cacheKey, space.task.props)
} catch (e) {
this.logger.error({ tag: this.updateCopyMoveTask.name, msg: `${e}` })
this.stopWatch(space.task.cacheKey)
diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.ts
new file mode 100644
index 00000000..71f30d77
--- /dev/null
+++ b/backend/src/applications/files/utils/tasks.spec.ts
@@ -0,0 +1,195 @@
+import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { FILE_OPERATION } from '../constants/operations'
+import * as filesUtils from './files'
+import { copyAbortable, createTaskTemporaryDir, isTaskCancellable, moveAbortable, SourceCleanupError, taskTemporaryPath } from './tasks'
+
+describe('abortable file tasks', () => {
+ const cacheKey = 'ftask-7-task-id'
+ let tmpDir: string
+ let srcPath: string
+ let dstPath: string
+ let stagingDir: string
+
+ beforeEach(async () => {
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))
+ srcPath = path.join(tmpDir, 'source.txt')
+ dstPath = path.join(tmpDir, 'destination.txt')
+ stagingDir = path.join(tmpDir, 'tasks')
+ })
+
+ afterEach(async () => {
+ await rm(tmpDir, { recursive: true, force: true })
+ vi.restoreAllMocks()
+ })
+
+ const mockCrossDevice = () => {
+ vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)
+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
+ }
+
+ it('marks intrinsically abortable operations as cancellable', async () => {
+ await expect(isTaskCancellable(FILE_OPERATION.COPY, '/source')).resolves.toBe(true)
+ await expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD, '/source')).resolves.toBe(true)
+ })
+
+ it('requires a cross-device destination for move and delete', async () => {
+ await writeFile(srcPath, 'content')
+ const lstatSpy = vi.spyOn(fs, 'lstat')
+ const statSpy = vi.spyOn(fs, 'stat')
+ lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)
+ statSpy.mockResolvedValueOnce({ dev: 2 } as any)
+
+ await expect(isTaskCancellable(FILE_OPERATION.MOVE, srcPath, dstPath)).resolves.toBe(true)
+ lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)
+ statSpy.mockResolvedValueOnce({ dev: 2 } as any)
+ await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath, dstPath)).resolves.toBe(true)
+ await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath)).resolves.toBe(false)
+ expect(lstatSpy).toHaveBeenCalledWith(srcPath)
+ })
+
+ it('uses a uniform task staging name for files and directories', async () => {
+ const filePath = taskTemporaryPath(tmpDir, cacheKey, '/destination/report.txt')
+ const directoryPath = await createTaskTemporaryDir(tmpDir, cacheKey, '/destination/archive')
+
+ expect(path.basename(filePath)).toBe(`.${cacheKey}-report.txt`)
+ expect(path.basename(directoryPath)).toBe(`.${cacheKey}-archive`)
+ expect((await fs.stat(directoryPath)).isDirectory()).toBe(true)
+ })
+
+ it('publishes a copied file only after the commit hook', async () => {
+ const signal = new AbortController().signal
+ let temporaryPath = ''
+ await writeFile(srcPath, 'content')
+
+ await copyAbortable(srcPath, dstPath, {
+ beforeCommit: async () => {
+ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')
+ },
+ cacheKey,
+ onTransferStart: (value) => {
+ temporaryPath = value
+ },
+ signal,
+ stagingDir
+ })
+
+ expect(path.dirname(temporaryPath)).toBe(stagingDir)
+ expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ })
+
+ it('cleans the temporary destination when copy is cancelled', async () => {
+ const controller = new AbortController()
+ await writeFile(srcPath, 'content')
+ controller.abort()
+
+ await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal: controller.signal, stagingDir })).rejects.toMatchObject({
+ name: 'AbortError'
+ })
+
+ await expect(access(dstPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('copies directly to a destination on another device', async () => {
+ const signal = new AbortController().signal
+ let watchedPath = ''
+ let destinationPrepared = false
+ await writeFile(srcPath, 'content')
+ mockCrossDevice()
+
+ await copyAbortable(srcPath, dstPath, {
+ beforeCommit: async () => {
+ destinationPrepared = true
+ },
+ cacheKey,
+ onTransferStart: (value) => {
+ expect(destinationPrepared).toBe(true)
+ watchedPath = value
+ },
+ signal,
+ stagingDir
+ })
+
+ expect(watchedPath).toBe(dstPath)
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('keeps an existing direct-copy destination when overwrite is rejected', async () => {
+ const signal = new AbortController().signal
+ await writeFile(srcPath, 'new content')
+ await writeFile(dstPath, 'existing content')
+ mockCrossDevice()
+
+ await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toMatchObject({ code: 'EEXIST' })
+
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
+ })
+
+ it('does not prepare a direct-copy destination when already cancelled', async () => {
+ const controller = new AbortController()
+ const beforeCommit = vi.fn()
+ await writeFile(srcPath, 'new content')
+ await writeFile(dstPath, 'existing content')
+ await fs.mkdir(stagingDir)
+ controller.abort()
+ mockCrossDevice()
+
+ await expect(
+ copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, overwrite: true, signal: controller.signal, stagingDir })
+ ).rejects.toMatchObject({ name: 'AbortError' })
+
+ expect(beforeCommit).not.toHaveBeenCalled()
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('existing content')
+ })
+
+ it('preserves a direct-copy error when destination cleanup also fails', async () => {
+ const signal = new AbortController().signal
+ const transferError = Object.assign(new Error('transfer failed'), { code: 'EIO' })
+ await fs.mkdir(stagingDir)
+ vi.spyOn(fs, 'lstat').mockImplementation(async (rPath) => {
+ if (rPath === stagingDir) return { dev: 1 } as any
+ throw transferError
+ })
+ vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(new Error('cleanup failed'))
+
+ await expect(copyAbortable(srcPath, dstPath, { cacheKey, signal, stagingDir })).rejects.toBe(transferError)
+ })
+
+ it('copies then removes the source for a cross-device move', async () => {
+ const signal = new AbortController().signal
+ await writeFile(srcPath, 'content')
+
+ await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })
+
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
+ it('reports source cleanup failure after keeping the published destination', async () => {
+ const signal = new AbortController().signal
+ const error = new Error('source cleanup failed')
+ await writeFile(srcPath, 'content')
+ vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)
+
+ const cleanupError = await moveAbortable(srcPath, dstPath, { cacheKey, crossDevice: true, signal, stagingDir })
+
+ expect(cleanupError).toBeInstanceOf(SourceCleanupError)
+ expect(cleanupError).toMatchObject({
+ cause: error,
+ dstPath,
+ message: 'Destination was published but the source could not be removed',
+ name: SourceCleanupError.name,
+ srcPath
+ })
+ await expect(readFile(srcPath, 'utf8')).resolves.toBe('content')
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ })
+})
diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.ts
new file mode 100644
index 00000000..b38748e8
--- /dev/null
+++ b/backend/src/applications/files/utils/tasks.ts
@@ -0,0 +1,203 @@
+import { createReadStream, createWriteStream } from 'node:fs'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { pipeline } from 'node:stream/promises'
+import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'
+import { FILE_OPERATION } from '../constants/operations'
+import { isPathExists, moveFiles, removeFiles } from './files'
+
+export interface TransferOptions {
+ // Runs immediately before the destination can be modified or published.
+ beforeCommit?: () => Promise<void>
+ // Identifies the owning task so cleanup can preserve its staging entry.
+ cacheKey: string
+ // Allows callers that already resolved the filesystem topology to skip another stat.
+ crossDevice?: boolean
+ // Provides the path whose progress must be watched during the transfer.
+ onTransferStart?: (temporaryPath: string) => void
+ overwrite?: boolean
+ signal: AbortSignal
+ // Preferred staging directory. Atomic publication requires it to share the destination filesystem.
+ stagingDir?: string
+}
+
+export interface CopyTaskOptions extends TransferOptions {
+ preserveTimestamps?: boolean
+ recursive?: boolean
+}
+
+// The destination is committed, but the obsolete source still requires manual or deferred cleanup.
+export class SourceCleanupError extends Error {
+ constructor(
+ readonly srcPath: string,
+ readonly dstPath: string,
+ options: ErrorOptions
+ ) {
+ super('Destination was published but the source could not be removed', options)
+ this.name = SourceCleanupError.name
+ }
+}
+
+export function taskTemporaryPrefix(cacheKey: string): string {
+ return `.${cacheKey}-`
+}
+
+export function taskTemporaryPath(parentPath: string, cacheKey: string, name: string): string {
+ return path.join(parentPath, `${taskTemporaryPrefix(cacheKey)}${path.basename(name)}`)
+}
+
+export async function createTaskTemporaryDir(parentPath: string, cacheKey: string, name: string): Promise<string> {
+ await fs.mkdir(parentPath, { recursive: true })
+ const temporaryPath = taskTemporaryPath(parentPath, cacheKey, name)
+ await fs.mkdir(temporaryPath)
+ return temporaryPath
+}
+
+async function copyEntry(srcPath: string, dstPath: string, recursive: boolean, preserveTimestamps: boolean, signal: AbortSignal): Promise<void> {
+ signal.throwIfAborted()
+ const stats = await fs.lstat(srcPath)
+ if (stats.isDirectory()) {
+ await fs.mkdir(dstPath)
+ if (recursive) {
+ for (const entry of await fs.readdir(srcPath)) {
+ await copyEntry(path.join(srcPath, entry), path.join(dstPath, entry), true, preserveTimestamps, signal)
+ }
+ }
+ } else if (stats.isSymbolicLink()) {
+ await fs.symlink(await fs.readlink(srcPath), dstPath)
+ } else {
+ await pipeline(
+ createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK }),
+ createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK }),
+ { signal }
+ )
+ }
+ if (!stats.isSymbolicLink()) {
+ await fs.chmod(dstPath, stats.mode)
+ if (preserveTimestamps) {
+ await fs.utimes(dstPath, stats.atime, stats.mtime)
+ }
+ }
+}
+
+async function existingParentPath(dstPath: string): Promise<string> {
+ let parentPath = path.dirname(dstPath)
+ while (!(await isPathExists(parentPath))) {
+ const nextParentPath = path.dirname(parentPath)
+ if (nextParentPath === parentPath) break
+ parentPath = nextParentPath
+ }
+ return parentPath
+}
+
+export async function isCrossDeviceMove(srcPath: string, dstPath: string): Promise<boolean> {
+ const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])
+ return srcStats.dev !== dstParentStats.dev
+}
+
+export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dstPath?: string): Promise<boolean> {
+ switch (type) {
+ case FILE_OPERATION.COPY:
+ case FILE_OPERATION.DOWNLOAD:
+ case FILE_OPERATION.COMPRESS:
+ case FILE_OPERATION.DECOMPRESS:
+ return true
+ case FILE_OPERATION.MOVE:
+ case FILE_OPERATION.DELETE:
+ if (!dstPath) return false
+ try {
+ return await isCrossDeviceMove(srcPath, dstPath)
+ } catch {
+ return false
+ }
+ default:
+ return false
+ }
+}
+
+async function prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {
+ if (!(await isPathExists(dstPath))) return
+ if (!overwrite) {
+ throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })
+ }
+ await removeFiles(dstPath)
+}
+
+async function cleanupAfterFailure(rPath: string): Promise<void> {
+ try {
+ await removeFiles(rPath)
+ } catch {
+ // Cleanup is best-effort and must not replace the transfer error.
+ }
+}
+
+export async function copyAbortable(srcPath: string, dstPath: string, options: CopyTaskOptions): Promise<void> {
+ const {
+ beforeCommit,
+ cacheKey,
+ onTransferStart,
+ overwrite = false,
+ preserveTimestamps = true,
+ recursive = true,
+ signal,
+ stagingDir = path.dirname(dstPath)
+ } = options
+ const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)
+ await fs.mkdir(stagingDir, { recursive: true })
+ const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)
+
+ // A staging file cannot be renamed atomically across filesystems. In that case,
+ // copy directly to the destination and remove partial data on handled failures.
+ // beforeCommit may already have moved an overwritten destination to trash when
+ // cancellation occurs; this recoverable side effect is accepted for external filesystems.
+ if (copyDirectlyToDestination) {
+ let transferStarted = false
+ try {
+ // Check before modifying the destination, then again before starting the long direct transfer.
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ signal.throwIfAborted()
+ await prepareDestination(dstPath, overwrite)
+ onTransferStart?.(dstPath)
+ transferStarted = true
+ await copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal)
+ } catch (e) {
+ if (transferStarted) {
+ await cleanupAfterFailure(dstPath)
+ }
+ throw e
+ }
+ return
+ }
+
+ // Same-filesystem staging keeps the destination untouched until the final rename.
+ try {
+ onTransferStart?.(temporaryPath)
+ await copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal)
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ await prepareDestination(dstPath, overwrite)
+ await fs.rename(temporaryPath, dstPath)
+ } catch (e) {
+ await cleanupAfterFailure(temporaryPath)
+ throw e
+ }
+}
+
+export async function moveAbortable(srcPath: string, dstPath: string, options: TransferOptions): Promise<SourceCleanupError | undefined> {
+ const { beforeCommit, cacheKey, onTransferStart, overwrite = false, signal, stagingDir } = options
+ const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))
+ if (!crossDevice) {
+ signal.throwIfAborted()
+ await beforeCommit?.()
+ await moveFiles(srcPath, dstPath, overwrite)
+ return
+ }
+ await copyAbortable(srcPath, dstPath, { beforeCommit, cacheKey, onTransferStart, overwrite, signal, stagingDir })
+ try {
+ await removeFiles(srcPath)
+ } catch (cause) {
+ // Publication is the commit point. The caller must update its persistent state before reporting this cleanup failure.
+ return new SourceCleanupError(srcPath, dstPath, { cause })
+ }
+}