Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
perf(files): batch delete completed tasks
johaven48ba4eef60a5
diff --git a/backend/src/applications/files/constants/routes.ts b/backend/src/applications/files/constants/routes.tsindex d234a0d5..ec9b6db2 100644--- a/backend/src/applications/files/constants/routes.ts+++ b/backend/src/applications/files/constants/routes.ts@@ -7,6 +7,7 @@ export const FILES_ROUTE = {TASK_OPERATION: 'task/operation',TASKS: 'tasks',TASKS_CANCEL: 'cancel',+ TASKS_DELETE: 'delete',TASKS_DOWNLOAD: 'download',TASKS_POLL: 'poll',RECENTS: 'recents',@@ -26,6 +27,7 @@ export const API_FILES_TASK_OPERATION_COMPRESS = `${API_FILES_TASK_OPERATION}/${export const API_FILES_TASK_OPERATION_DECOMPRESS = `${API_FILES_TASK_OPERATION}/${FILE_OPERATION.DECOMPRESS}`export const API_FILES_TASKS = `${FILES_ROUTE.BASE}/${FILES_ROUTE.TASKS}`export const API_FILES_TASKS_CANCEL = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_CANCEL}`+export const API_FILES_TASKS_DELETE = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_DELETE}`export const API_FILES_TASKS_DOWNLOAD = `${FILES_ROUTE.BASE}/${FILES_ROUTE.TASKS}/${FILES_ROUTE.TASKS_DOWNLOAD}`export const API_FILES_TASKS_POLL = `${API_FILES_TASKS}/${FILES_ROUTE.TASKS_POLL}`export const API_FILES_RECENTS = `${FILES_ROUTE.BASE}/${FILES_ROUTE.RECENTS}`diff --git a/backend/src/applications/files/dto/file-tasks-poll.dto.ts b/backend/src/applications/files/dto/file-tasks-poll.dto.tsdeleted file mode 100644index b800edb2..00000000--- a/backend/src/applications/files/dto/file-tasks-poll.dto.ts+++ /dev/null@@ -1,7 +0,0 @@-import { IsArray, IsString } from 'class-validator'--export class FileTasksPollDto {- @IsArray()- @IsString({ each: true })- trackedIds: string[]-}diff --git a/backend/src/applications/files/dto/file-tasks.dto.ts b/backend/src/applications/files/dto/file-tasks.dto.tsnew file mode 100644index 00000000..4fa8e157--- /dev/null+++ b/backend/src/applications/files/dto/file-tasks.dto.ts@@ -0,0 +1,14 @@+import { ArrayNotEmpty, IsArray, IsString } from 'class-validator'++export class FileTasksDeleteDto {+ @ArrayNotEmpty()+ @IsArray()+ @IsString({ each: true })+ taskIds: string[]+}++export class FileTasksPollDto {+ @IsArray()+ @IsString({ each: true })+ trackedIds: string[]+}diff --git a/backend/src/applications/files/files-tasks.controller.spec.ts b/backend/src/applications/files/files-tasks.controller.spec.tsindex 2c95f7fc..2446b2b2 100644--- a/backend/src/applications/files/files-tasks.controller.spec.ts+++ b/backend/src/applications/files/files-tasks.controller.spec.ts@@ -81,35 +81,35 @@ describe(FilesTasksController.name, () => {expect(result).toBe(expected)})- it('deleteTasks should call FilesTasksManager.deleteTasks with user and taskId and return the value', () => {- const user = { id: 'user-4', name: 'Alice' } as any- const taskId = 'task-del-1'- const expected = { deleted: true }+ it('deleteTasks should delegate deletion of all tasks', () => {+ const user = { id: 'user-5' } as any+ const expected = { deletedAll: true }filesTasksManager.deleteTasks.mockReturnValueOnce(expected)- const result = controller.deleteTasks(user, taskId)+ const result = controller.deleteTasks(user)expect(filesTasksManager.deleteTasks).toHaveBeenCalledTimes(1)- expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, 'task-del-1')+ expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user)expect(result).toBe(expected)})- it('deleteTasks without taskId must pass undefined', () => {- const user = { id: 'user-5' } as any- const expected = { deletedAll: true }+ it('deleteSelectedTasks should delegate task ids to FilesTasksManager', () => {+ const user = { id: 'user-6' } as any+ const dto = { taskIds: ['task-a', 'task-b'] }+ const expected = { deleted: true }filesTasksManager.deleteTasks.mockReturnValueOnce(expected)- const result = controller.deleteTasks(user)+ const result = controller.deleteSelectedTasks(user, dto)expect(filesTasksManager.deleteTasks).toHaveBeenCalledTimes(1)- expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, undefined)+ expect(filesTasksManager.deleteTasks).toHaveBeenCalledWith(user, dto.taskIds)expect(result).toBe(expected)})it('downloadTaskFile should delegate to FilesTasksManager.downloadArchive and return the StreamableFile', async () => {- const user = { id: 'user-6' } as any+ const user = { id: 'user-7' } as anyconst taskId = 'task-dl-42'const req = {} as anyconst res = {} as anydiff --git a/backend/src/applications/files/files-tasks.controller.ts b/backend/src/applications/files/files-tasks.controller.tsindex fb1c311b..241dae6c 100644--- a/backend/src/applications/files/files-tasks.controller.ts+++ b/backend/src/applications/files/files-tasks.controller.ts@@ -4,9 +4,9 @@ import { FastifySpaceRequest } from '../spaces/interfaces/space-request.interfacimport { GetUser } from '../users/decorators/user.decorator'import { UserModel } from '../users/models/user.model'import { API_FILES_TASKS, FILES_ROUTE } from './constants/routes'-import { FileTasksPollDto } from './dto/file-tasks-poll.dto'import type { FileTasksPollResponse } from './interfaces/file-task.interface'import { FilesTasksManager } from './services/tasks/files-tasks-manager.service'+import { FileTasksDeleteDto, FileTasksPollDto } from './dto/file-tasks.dto'@Controller(API_FILES_TASKS)export class FilesTasksController {@@ -22,9 +22,14 @@ export class FilesTasksController {return this.filesTasksManager.pollTasks(user.id, fileTasksPollDto.trackedIds)}- @Delete(':id?')- deleteTasks(@GetUser() user: UserModel, @Param('id') taskId?: string) {- return this.filesTasksManager.deleteTasks(user, taskId)+ @Delete()+ deleteTasks(@GetUser() user: UserModel) {+ return this.filesTasksManager.deleteTasks(user)+ }++ @Post(FILES_ROUTE.TASKS_DELETE)+ deleteSelectedTasks(@GetUser() user: UserModel, @Body() fileTasksDeleteDto: FileTasksDeleteDto) {+ return this.filesTasksManager.deleteTasks(user, fileTasksDeleteDto.taskIds)}@Post(`${FILES_ROUTE.TASKS_CANCEL}/:id`)diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.tsindex 30341272..95fab570 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts@@ -31,6 +31,7 @@ describe(FilesTasksManager.name, () => {mget: Mockincrement: Mockdel: Mock+ mdel: Mock}const flushPromises = async () => {@@ -82,7 +83,11 @@ describe(FilesTasksManager.name, () => {cacheStore.set(key, value)return value}),- del: vi.fn(async (key: string) => cacheStore.delete(key))+ del: vi.fn(async (key: string) => cacheStore.delete(key)),+ mdel: vi.fn(async (keys: string[]) => {+ for (const key of keys) cacheStore.delete(key)+ return true+ })}filesMethods = { doWork: vi.fn() }spacesManager = {@@ -549,11 +554,20 @@ describe(FilesTasksManager.name, () => {props: { compressInDirectory: false }})- await filesTasksManager.deleteTasks(user, 'task-ok')+ await filesTasksManager.deleteTasks(user, ['task-ok'])expect(removeFilesSpy).toHaveBeenCalledWith('/tmp/tasks/archive.tar')expect(stopWatchSpy).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)- expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)+ expect(cache.mget).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-22-task-ok`])+ expect(cache.mdel).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-22-task-ok`])+ })++ it('should not delete all tasks when selected task ids are empty', async () => {+ await filesTasksManager.deleteTasks({ id: 22 } as any, [])++ expect(cache.keys).not.toHaveBeenCalled()+ expect(cache.mget).not.toHaveBeenCalled()+ expect(cache.mdel).not.toHaveBeenCalled()})it('should ignore active tasks when deleting all tasks', async () => {@@ -580,10 +594,16 @@ describe(FilesTasksManager.name, () => {await filesTasksManager.deleteTasks(user)expect(cache.keys).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-31-*`)+ expect(cache.mget).toHaveBeenCalledWith([+ `${CACHE_TASK_PREFIX}-31-task-pending`,+ `${CACHE_TASK_PREFIX}-31-task-queued`,+ `${CACHE_TASK_PREFIX}-31-task-done`+ ])expect(removeFilesSpy).not.toHaveBeenCalled()expect(stopWatchSpy).toHaveBeenCalledTimes(1)- expect(cache.del).toHaveBeenCalledTimes(1)- expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-31-task-done`)+ expect(cache.del).not.toHaveBeenCalled()+ expect(cache.mdel).toHaveBeenCalledTimes(1)+ expect(cache.mdel).toHaveBeenCalledWith([`${CACHE_TASK_PREFIX}-31-task-done`])})it('should reject archive download when task is not applicable', async () => {diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.tsindex b0458869..461efd67 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts@@ -93,23 +93,20 @@ export class FilesTasksManager implements OnModuleDestroy {return { active, ended, missingIds: [...trackedTaskIds] }}- async deleteTasks(user: UserModel, taskId?: string): Promise<void> {- const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)- const keys: string[] = taskId ? [cacheKey] : await this.cache.keys(cacheKey)+ async deleteTasks(user: UserModel, taskIds?: string[]): Promise<void> {+ const keys =+ taskIds === undefined+ ? await this.cache.keys(FilesTasksManager.getCacheKey(user.id))+ : [...new Set(taskIds)].map((taskId: string) => FilesTasksManager.getCacheKey(user.id, taskId))if (!keys.length) return- for (const key of keys) {- const task: FileTask = await this.cache.get(key)- if (!task || this.isActiveStatus(task.status)) continue- if (task.props.compressInDirectory === false) {- // delete task file- const rPath = path.join(user.tasksPath, task.name)- removeFiles(rPath).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))- }- // clear watcher- this.filesTasksWatcher.stopWatch(key)- // remove from cache- this.cache.del(key).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))- }+ const tasks = (await this.cache.mget(keys)).filter(+ (task: FileTask | null | undefined): task is FileTask => task != null && !this.isActiveStatus(task.status)+ )+ if (!tasks.length) return++ const endedKeys = tasks.map((task: FileTask) => FilesTasksManager.getCacheKey(user.id, task.id))+ await Promise.all(tasks.map((task: FileTask, index: number) => this.cleanupTask(user, endedKeys[index], task)))+ await this.cache.mdel(endedKeys)}async cancelTask(userId: number, taskId: string): Promise<void> {@@ -148,6 +145,13 @@ export class FilesTasksManager implements OnModuleDestroy {return await sendFile.stream(req, res)}+ private async cleanupTask(user: UserModel, cacheKey: string, task: FileTask): Promise<void> {+ this.filesTasksWatcher.stopWatch(cacheKey)+ if (task.props.compressInDirectory === false) {+ await removeFiles(path.join(user.tasksPath, task.name)).catch((e: Error) => this.logger.error({ tag: this.deleteTasks.name, msg: `${e}` }))+ }+ }+private async storeTask(cacheKey: string, task: FileTask, status = FileTaskStatus.PENDING) {task.startedAt = currentTimeStamp(null, true)task.status = statusdiff --git a/frontend/src/app/applications/files/services/files-tasks.service.ts b/frontend/src/app/applications/files/services/files-tasks.service.tsindex 0787d777..7117230c 100644--- a/frontend/src/app/applications/files/services/files-tasks.service.ts+++ b/frontend/src/app/applications/files/services/files-tasks.service.ts@@ -1,7 +1,12 @@import { HttpClient, HttpErrorResponse } from '@angular/common/http'import { inject, Injectable } from '@angular/core'import { FILE_OPERATION } from '@sync-in-server/backend/src/applications/files/constants/operations'-import { API_FILES_TASKS, API_FILES_TASKS_CANCEL, API_FILES_TASKS_POLL } from '@sync-in-server/backend/src/applications/files/constants/routes'+import {+ API_FILES_TASKS,+ API_FILES_TASKS_CANCEL,+ API_FILES_TASKS_DELETE,+ API_FILES_TASKS_POLL+} from '@sync-in-server/backend/src/applications/files/constants/routes'import type { FileTasksPollResponse } from '@sync-in-server/backend/src/applications/files/interfaces/file-task.interface'import { FileTask, FileTaskStatus } from '@sync-in-server/backend/src/applications/files/models/file-task'import { SPACE_REPOSITORY } from '@sync-in-server/backend/src/applications/spaces/constants/spaces'@@ -103,9 +108,14 @@ export class FilesTasksService {})}- remove(task: FileTask) {- this.http.delete(`${API_FILES_TASKS}/${task.id}`).subscribe({- next: () => this.deleteTask(task.id, false),+ private removeSelectedTasks(tasks: FileTask[]) {+ const taskIds = [...new Set(tasks.map((task: FileTask) => task.id))]+ if (!taskIds.length) return+ this.http.post<void>(API_FILES_TASKS_DELETE, { taskIds }).subscribe({+ next: () => {+ const removedTaskIds = new Set(taskIds)+ this.store.filesEndedTasks.next(this.store.filesEndedTasks.getValue().filter((task: FileTask) => !removedTaskIds.has(task.id)))+ },error: (e) => console.error(e)})}@@ -326,7 +336,7 @@ export class FilesTasksService {private removeDeletedChildTasks(deleteTask: FileTask) {if (deleteTask.path.startsWith(SPACE_REPOSITORY.SHARES)) {- this.remove(deleteTask)+ this.removeSelectedTasks([deleteTask])} else if (deleteTask.path.startsWith(SPACE_REPOSITORY.TRASH)) {const deletedTrashPath = `${deleteTask.path}/${deleteTask.name}`const deletedFilesPath = `${SPACE_REPOSITORY.FILES}${deletedTrashPath.slice(SPACE_REPOSITORY.TRASH.length)}`@@ -340,10 +350,7 @@ export class FilesTasksService {taskPath.startsWith(`${deletedFilesPath}/`))})- for (const task of relatedTasks) {- this.remove(task)- }- this.remove(deleteTask)+ this.removeSelectedTasks([...relatedTasks, deleteTask])}}