Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): queue and limit concurrent tasks per user
johaven395f841a4755
diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.tsindex 714b792e..13093bb1 100644--- a/backend/src/applications/files/constants/cache.ts+++ b/backend/src/applications/files/constants/cache.ts@@ -2,6 +2,8 @@export const CACHE_TASK_PREFIX = 'ftask' as const// cache task cancellation key = `ftask-cancel-$(userId}-${taskId}` => booleanexport const CACHE_TASK_CANCEL_PREFIX = `${CACHE_TASK_PREFIX}-cancel` as const+// cache task user key = `ftask-user-$(userId}` => number of running FileTask+export const CACHE_TASK_USER_PREFIX = `${CACHE_TASK_PREFIX}-user` as constexport const CACHE_TASK_TTL = 86400 as const // one day// cache token key = `flock|token?:${uuid}|path:${path}|ownerId?:${number}|spaceId?:${number}|...props` => FileLockexport const CACHE_LOCK_PREFIX = 'flock' as constdiff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.tsindex a83606dd..919c2628 100644--- a/backend/src/applications/files/files.module.ts+++ b/backend/src/applications/files/files.module.ts@@ -16,6 +16,7 @@ import { FilesRecents } from './services/files-recents.service'import { FilesScheduler } from './services/files-scheduler.service'import { FilesSearchManager } from './services/files-search-manager.service'import { FilesTasksManager } from './services/files-tasks-manager.service'+import { FilesTasksQueue } from './services/files-tasks-queue.service'import { FilesEventManager } from './services/files-event-manager.service'import { FilesQuotaManager } from './services/files-quota-manager.service'import { FilesTrashRetention } from './services/files-trash-retention.service'@@ -32,6 +33,7 @@ import { FilesTrashRetention } from './services/files-trash-retention.service'FilesQueries,FilesLockManager,FilesTasksManager,+ FilesTasksQueue,FilesScheduler,FilesRecents,FilesContentParser,diff --git a/backend/src/applications/files/interfaces/file-task-queue.interface.ts b/backend/src/applications/files/interfaces/file-task-queue.interface.tsnew file mode 100644index 00000000..4eff224d--- /dev/null+++ b/backend/src/applications/files/interfaces/file-task-queue.interface.ts@@ -0,0 +1,19 @@+import type { SpaceEnv } from '../../spaces/models/space-env.model'+import type { UserModel } from '../../users/models/user.model'+import type { FileTask } from '../models/file-task'++export interface FileTaskQueueItem {+ cacheKey: string+ dto: any+ method: string+ space: SpaceEnv+ task: FileTask+ user: UserModel+}++export type FileTaskQueueStarter = (task: FileTaskQueueItem) => Promise<boolean>++export interface FileTaskQueueEntry {+ task: FileTaskQueueItem+ startTask: FileTaskQueueStarter+}diff --git a/backend/src/applications/files/models/file-task.ts b/backend/src/applications/files/models/file-task.tsindex 809396c9..d38ae6c0 100644--- a/backend/src/applications/files/models/file-task.ts+++ b/backend/src/applications/files/models/file-task.ts@@ -4,7 +4,8 @@ export enum FileTaskStatus {PENDING,SUCCESS,ERROR,- CANCELLED+ CANCELLED,+ QUEUED}export interface FileTaskProps {diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex acc93432..44b10dc8 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -4,6 +4,7 @@ import { isNotNull, sql } from 'drizzle-orm'import { unionAll } from 'drizzle-orm/mysql-core'import fs from 'node:fs/promises'import path from 'node:path'+import { currentTimeStamp } from '../../../common/shared'import { Cache } from '../../../infrastructure/cache/cache.service'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'@@ -11,7 +12,7 @@ import { getTablesWithFileIdColumn } from '../../../infrastructure/database/utilimport { USER_PATH, USER_ROLE } from '../../users/constants/user'import { UserModel } from '../../users/models/user.model'import { users } from '../../users/schemas/users.schema'-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX } from '../constants/cache'+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_USER_PREFIX } from '../constants/cache'import { FileTask, FileTaskStatus } from '../models/file-task'import { filesRecents } from '../schemas/files-recents.schema'import { files } from '../schemas/files.schema'@@ -260,6 +261,7 @@ export class FilesScheduler {try {let nb = 0let nbCancellationRequests = 0+ let nbUserTaskCounters = 0const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)for (const key of keys) {if (key.startsWith(`${CACHE_TASK_CANCEL_PREFIX}-`)) {@@ -267,20 +269,33 @@ export class FilesScheduler {nbCancellationRequests++continue}+ if (key.startsWith(`${CACHE_TASK_USER_PREFIX}-`)) {+ await this.cache.del(key)+ nbUserTaskCounters+++ continue+ }const task = await this.cache.get(key)- if (task && task.status === FileTaskStatus.PENDING) {+ if (task && this.isInterruptedTaskStatus(task.status)) {task.status = FileTaskStatus.ERRORtask.result = 'Interrupted'+ task.endedAt = currentTimeStamp(null, true)nb++this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))}}- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned, ${nbCancellationRequests} cancellation requests cleared` })+ this.logger.log({+ tag: this.cleanupInterruptedTasks.name,+ msg: `${nb} tasks cleaned, ${nbCancellationRequests} cancellation requests cleared, ${nbUserTaskCounters} user task counters cleared`+ })} catch (e) {this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })}}+ private isInterruptedTaskStatus(status: FileTaskStatus): boolean {+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED+ }+private async resetContentIndexingState(): Promise<void> {await this.filesContentIndexer.resetIndexingRuntimeState()this.logger.log({ tag: this.resetContentIndexingState.name, msg: `done` })diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.tsindex 6d848832..420749c9 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts@@ -6,10 +6,11 @@ import * as filesUtils from '../utils/files'import { FileTaskEvent } from '../events/file-events'import { SendFile } from '../utils/send-file'import { FILE_OPERATION } from '../constants/operations'-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'import { FileTaskStatus } from '../models/file-task'import { FilesMethods } from './files-methods.service'import { FilesTasksManager } from './files-tasks-manager.service'+import { FilesTasksQueue } from './files-tasks-queue.service'import { Mock } from 'vitest'describe(FilesTasksManager.name, () => {@@ -24,6 +25,7 @@ describe(FilesTasksManager.name, () => {get: Mockkeys: Mockmget: Mock+ increment: Mockdel: Mock}@@ -47,6 +49,11 @@ describe(FilesTasksManager.name, () => {get: vi.fn(async (key: string) => cacheStore.get(key)),keys: vi.fn(async (pattern: string) => [...cacheStore.keys()].filter((k) => createPatternRegex(pattern).test(k))),mget: vi.fn(async (keys: string[]) => keys.map((k) => cacheStore.get(k)).filter((v) => v !== undefined)),+ increment: vi.fn(async (key: string, amount = 1) => {+ const value = (Number(cacheStore.get(key)) || 0) + amount+ cacheStore.set(key, value)+ return value+ }),del: vi.fn(async (key: string) => cacheStore.delete(key))}filesMethods = {@@ -55,6 +62,7 @@ describe(FilesTasksManager.name, () => {module = await Test.createTestingModule({providers: [FilesTasksManager,+ FilesTasksQueue,{provide: Cache,useValue: cache@@ -97,6 +105,7 @@ describe(FilesTasksManager.name, () => {expect(FilesTasksManager.getCacheKey(10, 'task-1')).toBe(`${CACHE_TASK_PREFIX}-10-task-1`)expect(FilesTasksManager.getCacheKey(10)).toBe(`${CACHE_TASK_PREFIX}-10-*`)expect(FilesTasksManager.getCancellationCacheKey(10, 'task-1')).toBe(`${CACHE_TASK_CANCEL_PREFIX}-10-task-1`)+ expect(FilesTasksQueue.getUserRunningTasksCacheKey(10)).toBe(`${CACHE_TASK_USER_PREFIX}-10`)})it('should request cancellation through cache for a pending supported task', async () => {@@ -283,6 +292,89 @@ describe(FilesTasksManager.name, () => {expect(storedTask.result).toBe('operation failed')})+ it('should queue tasks when a user already has three running tasks', async () => {+ vi.spyOn(crypto, 'randomUUID')+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')+ filesMethods.doWork.mockImplementation(() => new Promise(() => undefined))+ const user = { id: 7 } as any++ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')++ expect(filesMethods.doWork).toHaveBeenCalledTimes(3)+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`).status).toBe(FileTaskStatus.PENDING)+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`).status).toBe(FileTaskStatus.QUEUED)+ expect(cacheStore.get(`${CACHE_TASK_USER_PREFIX}-7`)).toBe(3)+ })++ it('should start the next queued task when a running task completes', async () => {+ const resolveWork: (() => void)[] = []+ vi.spyOn(crypto, 'randomUUID')+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')+ filesMethods.doWork.mockImplementation(() => new Promise<void>((resolve) => resolveWork.push(resolve)))+ const user = { id: 7 } as any++ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')++ resolveWork[0]()+ await flushPromises()++ expect(filesMethods.doWork).toHaveBeenCalledTimes(4)+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`).status).toBe(FileTaskStatus.SUCCESS)+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`).status).toBe(FileTaskStatus.PENDING)+ expect(cacheStore.get(`${CACHE_TASK_USER_PREFIX}-7`)).toBe(3)+ })++ it('should cancel a queued cancellable task without starting it', async () => {+ vi.spyOn(crypto, 'randomUUID')+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')+ filesMethods.doWork.mockImplementation(() => new Promise(() => undefined))+ const user = { id: 7 } as any++ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')++ await filesTasksManager.cancelTask(7, '44444444-4444-4444-8444-444444444444')++ const storedTask = cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`)+ expect(filesMethods.doWork).toHaveBeenCalledTimes(3)+ expect(storedTask.status).toBe(FileTaskStatus.CANCELLED)+ expect(storedTask.result).toBe('Cancelled')+ })++ it('should cancel a cached queued cancellable task even when it is not in the local queue', async () => {+ cacheStore.set(`${CACHE_TASK_PREFIX}-8-task-queued`, {+ id: 'task-queued',+ type: FILE_OPERATION.DOWNLOAD,+ status: FileTaskStatus.QUEUED,+ props: {}+ })++ await filesTasksManager.cancelTask(8, 'task-queued')++ const storedTask = cacheStore.get(`${CACHE_TASK_PREFIX}-8-task-queued`)+ expect(storedTask.status).toBe(FileTaskStatus.CANCELLED)+ expect(storedTask.result).toBe('Cancelled')+ expect(storedTask.endedAt).toBeDefined()+ expect(cache.set).not.toHaveBeenCalledWith(`${CACHE_TASK_CANCEL_PREFIX}-8-task-queued`, true, CACHE_TASK_TTL)+ })+it('should return one task by id and throw when it does not exist', async () => {cacheStore.set(`${CACHE_TASK_PREFIX}-11-task-a`, { id: 'task-a' })@@ -319,7 +411,7 @@ describe(FilesTasksManager.name, () => {expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)})- it('should ignore pending tasks when deleting all tasks', async () => {+ it('should ignore active tasks when deleting all tasks', async () => {const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValue(undefined)const user = { id: 31, tasksPath: '/tmp/tasks' } as any@@ -328,6 +420,11 @@ describe(FilesTasksManager.name, () => {status: FileTaskStatus.PENDING,props: { compressInDirectory: false }})+ cacheStore.set(`${CACHE_TASK_PREFIX}-31-task-queued`, {+ id: 'task-queued',+ status: FileTaskStatus.QUEUED,+ props: { compressInDirectory: false }+ })cacheStore.set(`${CACHE_TASK_PREFIX}-31-task-done`, {id: 'task-done',name: 'done.tar',diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.tsindex c88e849c..dbd9890c 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts@@ -9,11 +9,13 @@ import { SpaceEnv } from '../../spaces/models/space-env.model'import { UserModel } from '../../users/models/user.model'import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'import { FILE_OPERATION } from '../constants/operations'+import { FileTaskEvent } from '../events/file-events'+import type { FileTaskQueueItem } from '../interfaces/file-task-queue.interface'import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'import { SendFile } from '../utils/send-file'import { FilesMethods } from './files-methods.service'-import { FileTaskEvent } from '../events/file-events'+import { FilesTasksQueue } from './files-tasks-queue.service'@Injectable()export class FilesTasksManager implements OnModuleDestroy {@@ -25,7 +27,8 @@ export class FilesTasksManager implements OnModuleDestroy {constructor(private readonly cache: Cache,- private readonly filesMethods: FilesMethods+ private readonly filesMethods: FilesMethods,+ private readonly filesTasksQueue: FilesTasksQueue) {FileTaskEvent.on('startWatch', this.onStartWatch)}@@ -52,23 +55,9 @@ export class FilesTasksManager implements OnModuleDestroy {const taskId: string = crypto.randomUUID()const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url))- await this.storeTask(cacheKey, newTask)- space.task = { cacheKey: cacheKey, props: {} }- const controller = this.watchCancellation(type, user.id, taskId, cacheKey)- const taskPromise = controller ? this.filesMethods[method](user, space, dto, controller.signal) : this.filesMethods[method](user, space, dto)- taskPromise- .then((data: any) => {- this.logger.debug({ tag: this.createTask.name, msg: `${newTask.name} : ${method} done` })- this.setTaskDone(cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) => this.logger.error({ tag: this.createTask.name, msg: `${e}` }))- })- .catch((e: HttpException | any) => {- this.logger.error({ tag: this.createTask.name, msg: `${newTask.name} : ${method} : ${e}` })- this.setTaskDone(- cacheKey,- controller?.signal.aborted ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,- controller?.signal.aborted ? 'Cancelled' : e.message- ).catch((e: Error) => this.logger.error({ tag: this.createTask.name, msg: `${e}` }))- })+ await this.storeTask(cacheKey, newTask, FileTaskStatus.QUEUED)+ space.task = { cacheKey, props: {} }+ await this.filesTasksQueue.enqueue(user.id, { cacheKey, dto, method, space, task: newTask, user }, (task) => this.startQueuedTask(task))return newTask}@@ -86,16 +75,11 @@ export class FilesTasksManager implements OnModuleDestroy {async deleteTasks(user: UserModel, taskId?: string): Promise<void> {const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)- let keys: string[]- if (taskId) {- keys = [cacheKey]- } else {- keys = await this.cache.keys(cacheKey)- }+ const keys: string[] = taskId ? [cacheKey] : await this.cache.keys(cacheKey)if (!keys.length) returnfor (const key of keys) {const task: FileTask = await this.cache.get(key)- if (!task || task.status === FileTaskStatus.PENDING) continue+ if (!task || this.isActiveStatus(task.status)) continueif (task.props.compressInDirectory === false) {// delete task fileconst rPath = path.join(user.tasksPath, task.name)@@ -114,9 +98,14 @@ export class FilesTasksManager implements OnModuleDestroy {if (!task) {throw new HttpException('Task not found', HttpStatus.NOT_FOUND)}- if (task.status !== FileTaskStatus.PENDING || !this.canCancel(task.type)) {+ if (!this.isActiveStatus(task.status) || !this.canCancel(task.type)) {throw new HttpException('Not applicable', HttpStatus.BAD_REQUEST)}+ if (task.status === FileTaskStatus.QUEUED) {+ this.filesTasksQueue.remove(userId, taskId)+ await this.setTaskDone(cacheKey, FileTaskStatus.CANCELLED, 'Cancelled')+ return+ }const isStored = await this.cache.set(FilesTasksManager.getCancellationCacheKey(userId, taskId), true, CACHE_TASK_TTL)if (!isStored) {throw new HttpException('Unable to cancel task', HttpStatus.INTERNAL_SERVER_ERROR)@@ -139,9 +128,9 @@ export class FilesTasksManager implements OnModuleDestroy {return await sendFile.stream(req, res)}- private async storeTask(cacheKey: string, task: FileTask) {+ private async storeTask(cacheKey: string, task: FileTask, status = FileTaskStatus.PENDING) {task.startedAt = currentTimeStamp(null, true)- task.status = FileTaskStatus.PENDING+ task.status = statustry {await this.cache.set(cacheKey, task, CACHE_TASK_TTL)} catch (e) {@@ -170,6 +159,52 @@ export class FilesTasksManager implements OnModuleDestroy {.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))}+ private async startQueuedTask(task: FileTaskQueueItem): Promise<boolean> {+ const storedTask: FileTask = await this.cache.get(task.cacheKey)+ if (!storedTask || storedTask.status !== FileTaskStatus.QUEUED) return false+ task.task.status = FileTaskStatus.PENDING+ task.task.startedAt = currentTimeStamp(null, true)+ task.space.task = { cacheKey: task.cacheKey, props: task.task.props }+ await this.cache.set(task.cacheKey, task.task, CACHE_TASK_TTL)+ this.runTask(task)+ return true+ }++ private runTask(task: FileTaskQueueItem): void {+ const controller = this.watchCancellation(task.task.type, task.user.id, task.task.id, task.cacheKey)+ const taskPromise = controller+ ? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)+ : this.filesMethods[task.method](task.user, task.space, task.dto)+ taskPromise+ .then((data: any) => {+ this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })+ this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) =>+ this.logger.error({ tag: this.runTask.name, msg: `${e}` })+ )+ })+ .catch((e: HttpException | any) => {+ this.logger.error({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} : ${e}` })+ this.completeTask(+ task.user.id,+ task.cacheKey,+ controller?.signal.aborted ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,+ controller?.signal.aborted ? 'Cancelled' : e.message+ ).catch((e: Error) => this.logger.error({ tag: this.runTask.name, msg: `${e}` }))+ })+ }++ private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {+ try {+ await this.setTaskDone(cacheKey, status, result)+ } finally {+ await this.filesTasksQueue.releaseAndDrain(userId)+ }+ }++ private isActiveStatus(status: FileTaskStatus): boolean {+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED+ }+private async updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {let ftask: FileTask = await this.cache.get(cacheKey)if (ftask) {diff --git a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts b/backend/src/applications/files/services/files-tasks-queue.service.spec.tsnew file mode 100644index 00000000..3f45541d--- /dev/null+++ b/backend/src/applications/files/services/files-tasks-queue.service.spec.ts@@ -0,0 +1,137 @@+import { Mock } from 'vitest'+import { Cache } from '../../../infrastructure/cache/cache.service'+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'+import type { FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'+import { FilesTasksQueue } from './files-tasks-queue.service'++describe(FilesTasksQueue.name, () => {+ const userId = 42+ const counterKey = `${CACHE_TASK_USER_PREFIX}-${userId}`+ let counters: Map<string, number>+ let increment: Mock+ let queue: FilesTasksQueue++ const createTask = (id: string): FileTaskQueueItem =>+ ({+ cacheKey: `ftask-${userId}-${id}`,+ task: { id }+ }) as FileTaskQueueItem++ beforeEach(() => {+ counters = new Map<string, number>()+ increment = vi.fn(async (key: string, amount = 1, _ttl?: number, minimum?: number) => {+ const value = Math.max((counters.get(key) ?? 0) + amount, minimum ?? Number.NEGATIVE_INFINITY)+ counters.set(key, value)+ return value+ })+ queue = new FilesTasksQueue({ increment } as unknown as Cache)+ })++ afterEach(() => {+ queue.stop()+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it('should build the shared running tasks cache key', () => {+ expect(FilesTasksQueue.getUserRunningTasksCacheKey(userId)).toBe(counterKey)+ })++ it('should start at most three tasks per user and queue the next one', async () => {+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)+ const tasks = ['task-1', 'task-2', 'task-3', 'task-4'].map(createTask)++ for (const task of tasks) {+ await queue.enqueue(userId, task, startTask)+ }++ expect(startTask).toHaveBeenCalledTimes(3)+ expect(startTask.mock.calls.map(([task]) => task.task.id)).toEqual(['task-1', 'task-2', 'task-3'])+ expect(counters.get(counterKey)).toBe(3)+ })++ it('should apply the running task limit independently for each user', async () => {+ const secondUserId = 84+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)++ for (const currentUserId of [userId, secondUserId]) {+ for (const id of ['task-1', 'task-2', 'task-3']) {+ await queue.enqueue(currentUserId, createTask(`${currentUserId}-${id}`), startTask)+ }+ }++ expect(startTask).toHaveBeenCalledTimes(6)+ expect(counters.get(counterKey)).toBe(3)+ expect(counters.get(`${CACHE_TASK_USER_PREFIX}-${secondUserId}`)).toBe(3)+ })++ it('should start the next queued task after releasing a slot', async () => {+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)++ for (const id of ['task-1', 'task-2', 'task-3', 'task-4']) {+ await queue.enqueue(userId, createTask(id), startTask)+ }+ await queue.releaseAndDrain(userId)++ expect(startTask).toHaveBeenCalledTimes(4)+ expect(startTask).toHaveBeenLastCalledWith(expect.objectContaining({ task: expect.objectContaining({ id: 'task-4' }) }))+ expect(counters.get(counterKey)).toBe(3)+ })++ it('should remove a queued task without starting it', async () => {+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)++ for (const id of ['task-1', 'task-2', 'task-3', 'task-4']) {+ await queue.enqueue(userId, createTask(id), startTask)+ }+ queue.remove(userId, 'task-4')+ await queue.releaseAndDrain(userId)++ expect(startTask).toHaveBeenCalledTimes(3)+ expect(counters.get(counterKey)).toBe(2)+ })++ it('should release the claimed slot when a task cannot start', async () => {+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(false)++ await queue.enqueue(userId, createTask('task-1'), startTask)++ expect(startTask).toHaveBeenCalledOnce()+ expect(counters.get(counterKey)).toBe(0)+ expect(increment).toHaveBeenLastCalledWith(counterKey, -1, CACHE_TASK_TTL, 0)+ })++ it('should release the claimed slot when the task starter throws', async () => {+ const error = new Error('start failed')+ const startTask = vi.fn<FileTaskQueueStarter>().mockRejectedValue(error)+ const loggerSpy = vi.spyOn((queue as any).logger, 'error').mockImplementation(() => undefined)++ await queue.enqueue(userId, createTask('task-1'), startTask)++ expect(loggerSpy).toHaveBeenCalledWith({ tag: 'drain', msg: `${error}` })+ expect(counters.get(counterKey)).toBe(0)+ })++ it('should never decrement the shared counter below zero', async () => {+ await queue.releaseAndDrain(userId)+ await queue.releaseAndDrain(userId)++ expect(counters.get(counterKey)).toBe(0)+ expect(increment).toHaveBeenLastCalledWith(counterKey, -1, CACHE_TASK_TTL, 0)+ })++ it('should periodically retry queued tasks when a shared slot becomes available', async () => {+ vi.useFakeTimers()+ counters.set(counterKey, 3)+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)++ await queue.enqueue(userId, createTask('task-1'), startTask)+ expect(startTask).not.toHaveBeenCalled()++ counters.set(counterKey, 2)+ await vi.advanceTimersByTimeAsync(1000)++ expect(startTask).toHaveBeenCalledOnce()+ expect(counters.get(counterKey)).toBe(3)+ })+})diff --git a/backend/src/applications/files/services/files-tasks-queue.service.ts b/backend/src/applications/files/services/files-tasks-queue.service.tsnew file mode 100644index 00000000..75f02f87--- /dev/null+++ b/backend/src/applications/files/services/files-tasks-queue.service.ts@@ -0,0 +1,117 @@+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'+import { Cache } from '../../../infrastructure/cache/cache.service'+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'+import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'++@Injectable()+export class FilesTasksQueue implements OnModuleDestroy {+ private readonly logger = new Logger(FilesTasksQueue.name)+ private readonly maxRunningTasksPerUser = 3+ private readonly drainInterval = 1000+ private drainingUsers = new Set<number>()+ private queuedTasks: Record<number, FileTaskQueueEntry[]> = {}+ private queuedTasksDrainInterval: NodeJS.Timeout | undefined++ constructor(private readonly cache: Cache) {}++ static getUserRunningTasksCacheKey(userId: number): string {+ return `${CACHE_TASK_USER_PREFIX}-${userId}`+ }++ onModuleDestroy(): void {+ this.stop()+ }++ async enqueue(userId: number, task: FileTaskQueueItem, startTask: FileTaskQueueStarter): Promise<void> {+ ;(this.queuedTasks[userId] ??= []).push({ task, startTask })+ this.startQueueDrainInterval()+ await this.drain(userId)+ }++ remove(userId: number, taskId: string): void {+ const queue = this.queuedTasks[userId]+ if (!queue) return+ const taskIndex = queue.findIndex((entry: FileTaskQueueEntry) => entry.task.task.id === taskId)+ if (taskIndex !== -1) {+ queue.splice(taskIndex, 1)+ }+ if (!queue.length) {+ delete this.queuedTasks[userId]+ this.stopQueueDrainIntervalIfIdle()+ }+ }++ async releaseAndDrain(userId: number): Promise<void> {+ await this.releaseUserRunningTaskSlot(userId)+ await this.drain(userId)+ }++ stop(): void {+ if (!this.queuedTasksDrainInterval) return+ clearInterval(this.queuedTasksDrainInterval)+ this.queuedTasksDrainInterval = undefined+ }++ private async drain(userId: number): Promise<void> {+ if (this.drainingUsers.has(userId)) return+ this.drainingUsers.add(userId)+ try {+ const queue = this.queuedTasks[userId]+ if (!queue?.length) {+ this.stopQueueDrainIntervalIfIdle()+ return+ }+ while (queue.length) {+ if (!(await this.claimUserRunningTaskSlot(userId))) break+ const entry = queue.shift()+ try {+ if (entry && (await entry.startTask(entry.task))) continue+ } catch (e) {+ this.logger.error({ tag: this.drain.name, msg: `${e}` })+ }+ await this.releaseUserRunningTaskSlot(userId)+ }+ if (!queue.length) {+ delete this.queuedTasks[userId]+ this.stopQueueDrainIntervalIfIdle()+ }+ } finally {+ this.drainingUsers.delete(userId)+ }+ }++ private async claimUserRunningTaskSlot(userId: number): Promise<boolean> {+ const cacheKey = FilesTasksQueue.getUserRunningTasksCacheKey(userId)+ const runningTasks = await this.cache.increment(cacheKey, 1, CACHE_TASK_TTL)+ if (runningTasks <= this.maxRunningTasksPerUser) {+ return true+ }+ await this.releaseUserRunningTaskSlot(userId)+ return false+ }++ private async releaseUserRunningTaskSlot(userId: number): Promise<void> {+ const cacheKey = FilesTasksQueue.getUserRunningTasksCacheKey(userId)+ await this.cache.increment(cacheKey, -1, CACHE_TASK_TTL, 0)+ }++ private startQueueDrainInterval(): void {+ if (this.queuedTasksDrainInterval) return+ this.queuedTasksDrainInterval = setInterval(() => {+ this.drainAll().catch((e: Error) => this.logger.error({ tag: this.drainAll.name, msg: `${e}` }))+ }, this.drainInterval)+ this.queuedTasksDrainInterval.unref()+ }++ private stopQueueDrainIntervalIfIdle(): void {+ if (Object.keys(this.queuedTasks).length) return+ this.stop()+ }++ private async drainAll(): Promise<void> {+ for (const userId of Object.keys(this.queuedTasks)) {+ await this.drain(Number(userId))+ }+ this.stopQueueDrainIntervalIfIdle()+ }+}