Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

perf(files): batch delete completed tasks

johaven48ba4eef60a5
diff --git a/backend/src/applications/files/constants/routes.ts b/backend/src/applications/files/constants/routes.ts
index d234a0d5..ec9b6db2 100644
--- a/backend/src/applications/files/constants/routes.ts
+++ b/backend/src/applications/files/constants/routes.ts
@@ -7,6 +7,7 @@ export const FILES_ROUTE = {
TASK_OPERATION: 'task/operation',
TASKS: 'tasks',
TASKS_CANCEL: 'cancel',
+ TASKS_DELETE: 'delete',
TASKS_DOWNLOAD: 'download',
TASKS_POLL: 'poll',
RECENTS: 'recents',
@@ -26,6 +27,7 @@ export const API_FILES_TASK_OPERATION_COMPRESS = `${API_FILES_TASK_OPERATION}/${
export const API_FILES_TASK_OPERATION_DECOMPRESS = `${API_FILES_TASK_OPERATION}/${FILE_OPERATION.DECOMPRESS}`
export const API_FILES_TASKS = `${FILES_ROUTE.BASE}/${FILES_ROUTE.TASKS}`
export const API_FILES_TASKS_CANCEL = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_CANCEL}`
+export const API_FILES_TASKS_DELETE = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_DELETE}`
export const API_FILES_TASKS_DOWNLOAD = `${FILES_ROUTE.BASE}/${FILES_ROUTE.TASKS}/${FILES_ROUTE.TASKS_DOWNLOAD}`
export const API_FILES_TASKS_POLL = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_POLL}`
export const API_FILES_RECENTS = `${FILES_ROUTE.BASE}/${FILES_ROUTE.RECENTS}`
diff --git a/backend/src/applications/files/dto/file-tasks-poll.dto.ts b/backend/src/applications/files/dto/file-tasks-poll.dto.ts
deleted file mode 100644
index b800edb2..00000000
--- a/backend/src/applications/files/dto/file-tasks-poll.dto.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { IsArray, IsString } from 'class-validator'
-
-export class FileTasksPollDto {
- @IsArray()
- @IsString({ each: true })
- trackedIds: string[]
-}
diff --git a/backend/src/applications/files/dto/file-tasks.dto.ts b/backend/src/applications/files/dto/file-tasks.dto.ts
new file mode 100644
index 00000000..4fa8e157
--- /dev/null
+++ b/backend/src/applications/files/dto/file-tasks.dto.ts
@@ -0,0 +1,14 @@
+import { ArrayNotEmpty, IsArray, IsString } from 'class-validator'
+
+export class FileTasksDeleteDto {
+ @ArrayNotEmpty()
+ @IsArray()
+ @IsString({ each: true })
+ taskIds: string[]
+}
+
+export class FileTasksPollDto {
+ @IsArray()
+ @IsString({ each: true })
+ trackedIds: string[]
+}
diff --git a/backend/src/applications/files/files-tasks.controller.spec.ts b/backend/src/applications/files/files-tasks.controller.spec.ts
index 2c95f7fc..2446b2b2 100644
--- a/backend/src/applications/files/files-tasks.controller.spec.ts
+++ b/backend/src/applications/files/files-tasks.controller.spec.ts
@@ -81,35 +81,35 @@ describe(FilesTasksController.name, () => {
expect(result).toBe(expected)
})
- it('deleteTasks should call FilesTasksManager.deleteTasks with user and taskId and return the value', () => {
- const user = { id: 'user-4', name: 'Alice' } as any
- const taskId = 'task-del-1'
- const expected = { deleted: true }
+ it('deleteTasks should delegate deletion of all tasks', () => {
+ const user = { id: 'user-5' } as any
+ const expected = { deletedAll: true }
filesTasksManager.deleteTasks.mockReturnValueOnce(expected)
- const result = controller.deleteTasks(user, taskId)
+ const result = controller.deleteTasks(user)
expect(filesTasksManager.deleteTasks).toHaveBeenCalledTimes(1)
- expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, 'task-del-1')
+ expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user)
expect(result).toBe(expected)
})
- it('deleteTasks without taskId must pass undefined', () => {
- const user = { id: 'user-5' } as any
- const expected = { deletedAll: true }
+ it('deleteSelectedTasks should delegate task ids to FilesTasksManager', () => {
+ const user = { id: 'user-6' } as any
+ const dto = { taskIds: ['task-a', 'task-b'] }
+ const expected = { deleted: true }
filesTasksManager.deleteTasks.mockReturnValueOnce(expected)
- const result = controller.deleteTasks(user)
+ const result = controller.deleteSelectedTasks(user, dto)
expect(filesTasksManager.deleteTasks).toHaveBeenCalledTimes(1)
- expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, undefined)
+ expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, dto.taskIds)
expect(result).toBe(expected)
})
it('downloadTaskFile should delegate to FilesTasksManager.downloadArchive and return the StreamableFile', async () => {
- const user = { id: 'user-6' } as any
+ const user = { id: 'user-7' } as any
const taskId = 'task-dl-42'
const req = {} as any
const res = {} as any
diff --git a/backend/src/applications/files/files-tasks.controller.ts b/backend/src/applications/files/files-tasks.controller.ts
index fb1c311b..241dae6c 100644
--- a/backend/src/applications/files/files-tasks.controller.ts
+++ b/backend/src/applications/files/files-tasks.controller.ts
@@ -4,9 +4,9 @@ import { FastifySpaceRequest } from '../spaces/interfaces/space-request.interfac
import { GetUser } from '../users/decorators/user.decorator'
import { UserModel } from '../users/models/user.model'
import { API_FILES_TASKS, FILES_ROUTE } from './constants/routes'
-import { FileTasksPollDto } from './dto/file-tasks-poll.dto'
import type { FileTasksPollResponse } from './interfaces/file-task.interface'
import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'
+import { FileTasksDeleteDto, FileTasksPollDto } from './dto/file-tasks.dto'
@Controller(API_FILES_TASKS)
export class FilesTasksController {
@@ -22,9 +22,14 @@ export class FilesTasksController {
return this.filesTasksManager.pollTasks(user.id, fileTasksPollDto.trackedIds)
}
- @Delete(':id?')
- deleteTasks(@GetUser() user: UserModel, @Param('id') taskId?: string) {
- return this.filesTasksManager.deleteTasks(user, taskId)
+ @Delete()
+ deleteTasks(@GetUser() user: UserModel) {
+ return this.filesTasksManager.deleteTasks(user)
+ }
+
+ @Post(FILES_ROUTE.TASKS_DELETE)
+ deleteSelectedTasks(@GetUser() user: UserModel, @Body() fileTasksDeleteDto: FileTasksDeleteDto) {
+ return this.filesTasksManager.deleteTasks(user, fileTasksDeleteDto.taskIds)
}
@Post(`${FILES_ROUTE.TASKS_CANCEL}/:id`)
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
index 30341272..95fab570 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
@@ -31,6 +31,7 @@ describe(FilesTasksManager.name, () => {
mget: Mock
increment: Mock
del: Mock
+ mdel: Mock
}
const flushPromises = async () => {
@@ -82,7 +83,11 @@ describe(FilesTasksManager.name, () => {
cacheStore.set(key, value)
return value
}),
- del: vi.fn(async (key: string) => cacheStore.delete(key))
+ del: vi.fn(async (key: string) => cacheStore.delete(key)),
+ mdel: vi.fn(async (keys: string[]) => {
+ for (const key of keys) cacheStore.delete(key)
+ return true
+ })
}
filesMethods = { doWork: vi.fn() }
spacesManager = {
@@ -549,11 +554,20 @@ describe(FilesTasksManager.name, () => {
props: { compressInDirectory: false }
})
- await filesTasksManager.deleteTasks(user, 'task-ok')
+ await filesTasksManager.deleteTasks(user, ['task-ok'])
expect(removeFilesSpy).toHaveBeenCalledWith('/tmp/tasks/archive.tar')
expect(stopWatchSpy).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)
- expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)
+ expect(cache.mget).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-22-task-ok`])
+ expect(cache.mdel).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-22-task-ok`])
+ })
+
+ it('should not delete all tasks when selected task ids are empty', async () => {
+ await filesTasksManager.deleteTasks({ id: 22 } as any, [])
+
+ expect(cache.keys).not.toHaveBeenCalled()
+ expect(cache.mget).not.toHaveBeenCalled()
+ expect(cache.mdel).not.toHaveBeenCalled()
})
it('should ignore active tasks when deleting all tasks', async () => {
@@ -580,10 +594,16 @@ describe(FilesTasksManager.name, () => {
await filesTasksManager.deleteTasks(user)
expect(cache.keys).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-31-*`)
+ expect(cache.mget).toHaveBeenCalledWith([
+ `${CACHE_TASK_PREFIX}-31-task-pending`,
+ `${CACHE_TASK_PREFIX}-31-task-queued`,
+ `${CACHE_TASK_PREFIX}-31-task-done`
+ ])
expect(removeFilesSpy).not.toHaveBeenCalled()
expect(stopWatchSpy).toHaveBeenCalledTimes(1)
- expect(cache.del).toHaveBeenCalledTimes(1)
- expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-31-task-done`)
+ expect(cache.del).not.toHaveBeenCalled()
+ expect(cache.mdel).toHaveBeenCalledTimes(1)
+ expect(cache.mdel).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-31-task-done`])
})
it('should reject archive download when task is not applicable', async () => {
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
index b0458869..461efd67 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
@@ -93,23 +93,20 @@ export class FilesTasksManager implements OnModuleDestroy {
return { active, ended, missingIds: [...trackedTaskIds] }
}
- async deleteTasks(user: UserModel, taskId?: string): Promise<void> {
- const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)
- const keys: string[] = taskId ? [cacheKey] : await this.cache.keys(cacheKey)
+ async deleteTasks(user: UserModel, taskIds?: string[]): Promise<void> {
+ const keys =
+ taskIds === undefined
+ ? await this.cache.keys(FilesTasksManager.getCacheKey(user.id))
+ : [...new Set(taskIds)].map((taskId: string) => FilesTasksManager.getCacheKey(user.id, taskId))
if (!keys.length) return
- for (const key of keys) {
- const task: FileTask = await this.cache.get(key)
- if (!task || this.isActiveStatus(task.status)) continue
- if (task.props.compressInDirectory === false) {
- // delete task file
- const rPath = path.join(user.tasksPath, task.name)
- removeFiles(rPath).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))
- }
- // clear watcher
- this.filesTasksWatcher.stopWatch(key)
- // remove from cache
- this.cache.del(key).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))
- }
+ const tasks = (await this.cache.mget(keys)).filter(
+ (task: FileTask | null | undefined): task is FileTask => task != null && !this.isActiveStatus(task.status)
+ )
+ if (!tasks.length) return
+
+ const endedKeys = tasks.map((task: FileTask) => FilesTasksManager.getCacheKey(user.id, task.id))
+ await Promise.all(tasks.map((task: FileTask, index: number) => this.cleanupTask(user, endedKeys[index], task)))
+ await this.cache.mdel(endedKeys)
}
async cancelTask(userId: number, taskId: string): Promise<void> {
@@ -148,6 +145,13 @@ export class FilesTasksManager implements OnModuleDestroy {
return await sendFile.stream(req, res)
}
+ private async cleanupTask(user: UserModel, cacheKey: string, task: FileTask): Promise<void> {
+ this.filesTasksWatcher.stopWatch(cacheKey)
+ if (task.props.compressInDirectory === false) {
+ await removeFiles(path.join(user.tasksPath, task.name)).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))
+ }
+ }
+
private async storeTask(cacheKey: string, task: FileTask, status = FileTaskStatus.PENDING) {
task.startedAt = currentTimeStamp(null, true)
task.status = status
diff --git a/frontend/src/app/applications/files/services/files-tasks.service.ts b/frontend/src/app/applications/files/services/files-tasks.service.ts
index 0787d777..7117230c 100644
--- a/frontend/src/app/applications/files/services/files-tasks.service.ts
+++ b/frontend/src/app/applications/files/services/files-tasks.service.ts
@@ -1,7 +1,12 @@
import { HttpClient, HttpErrorResponse } from '@angular/common/http'
import { inject, Injectable } from '@angular/core'
import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'
-import { API_FILES_TASKS, API_FILES_TASKS_CANCEL, API_FILES_TASKS_POLL } from '@sync-in-server/backend/src/applications/files/constants/routes'
+import {
+ API_FILES_TASKS,
+ API_FILES_TASKS_CANCEL,
+ API_FILES_TASKS_DELETE,
+ API_FILES_TASKS_POLL
+} from '@sync-in-server/backend/src/applications/files/constants/routes'
import type { FileTasksPollResponse } from '@sync-in-server/backend/src/applications/files/interfaces/file-task.interface'
import { FileTask, FileTaskStatus } from '@sync-in-server/backend/src/applications/files/models/file-task'
import { SPACE_REPOSITORY } from '@sync-in-server/backend/src/applications/spaces/constants/spaces'
@@ -103,9 +108,14 @@ export class FilesTasksService {
})
}
- remove(task: FileTask) {
- this.http.delete(`${API_FILES_TASKS}/${task.id}`).subscribe({
- next: () => this.deleteTask(task.id, false),
+ private removeSelectedTasks(tasks: FileTask[]) {
+ const taskIds = [...new Set(tasks.map((task: FileTask) => task.id))]
+ if (!taskIds.length) return
+ this.http.post<void>(API_FILES_TASKS_DELETE, { taskIds }).subscribe({
+ next: () => {
+ const removedTaskIds = new Set(taskIds)
+ this.store.filesEndedTasks.next(this.store.filesEndedTasks.getValue().filter((task: FileTask) => !removedTaskIds.has(task.id)))
+ },
error: (e) => console.error(e)
})
}
@@ -326,7 +336,7 @@ export class FilesTasksService {
private removeDeletedChildTasks(deleteTask: FileTask) {
if (deleteTask.path.startsWith(SPACE_REPOSITORY.SHARES)) {
- this.remove(deleteTask)
+ this.removeSelectedTasks([deleteTask])
} else if (deleteTask.path.startsWith(SPACE_REPOSITORY.TRASH)) {
const deletedTrashPath = `${deleteTask.path}/${deleteTask.name}`
const deletedFilesPath = `${SPACE_REPOSITORY.FILES}${deletedTrashPath.slice(SPACE_REPOSITORY.TRASH.length)}`
@@ -340,10 +350,7 @@ export class FilesTasksService {
taskPath.startsWith(`${deletedFilesPath}/`)
)
})
- for (const task of relatedTasks) {
- this.remove(task)
- }
- this.remove(deleteTask)
+ this.removeSelectedTasks([...relatedTasks, deleteTask])
}
}