Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): queue and limit concurrent tasks per user

johaven395f841a4755
diff --git a/backend/src/applications/files/constants/cache.ts b/backend/src/applications/files/constants/cache.ts
index 714b792e..13093bb1 100644
--- a/backend/src/applications/files/constants/cache.ts
+++ b/backend/src/applications/files/constants/cache.ts
@@ -2,6 +2,8 @@
export const CACHE_TASK_PREFIX = 'ftask' as const
// cache task cancellation key = `ftask-cancel-$(userId}-${taskId}` => boolean
export const CACHE_TASK_CANCEL_PREFIX = `${CACHE_TASK_PREFIX}-cancel` as const
+// cache task user key = `ftask-user-$(userId}` => number of running FileTask
+export const CACHE_TASK_USER_PREFIX = `${CACHE_TASK_PREFIX}-user` as const
export const CACHE_TASK_TTL = 86400 as const // one day
// cache token key = `flock|token?:${uuid}|path:${path}|ownerId?:${number}|spaceId?:${number}|...props` => FileLock
export const CACHE_LOCK_PREFIX = 'flock' as const
diff --git a/backend/src/applications/files/files.module.ts b/backend/src/applications/files/files.module.ts
index a83606dd..919c2628 100644
--- a/backend/src/applications/files/files.module.ts
+++ b/backend/src/applications/files/files.module.ts
@@ -16,6 +16,7 @@ import { FilesRecents } from './services/files-recents.service'
import { FilesScheduler } from './services/files-scheduler.service'
import { FilesSearchManager } from './services/files-search-manager.service'
import { FilesTasksManager } from './services/files-tasks-manager.service'
+import { FilesTasksQueue } from './services/files-tasks-queue.service'
import { FilesEventManager } from './services/files-event-manager.service'
import { FilesQuotaManager } from './services/files-quota-manager.service'
import { FilesTrashRetention } from './services/files-trash-retention.service'
@@ -32,6 +33,7 @@ import { FilesTrashRetention } from './services/files-trash-retention.service'
FilesQueries,
FilesLockManager,
FilesTasksManager,
+ FilesTasksQueue,
FilesScheduler,
FilesRecents,
FilesContentParser,
diff --git a/backend/src/applications/files/interfaces/file-task-queue.interface.ts b/backend/src/applications/files/interfaces/file-task-queue.interface.ts
new file mode 100644
index 00000000..4eff224d
--- /dev/null
+++ b/backend/src/applications/files/interfaces/file-task-queue.interface.ts
@@ -0,0 +1,19 @@
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+import type { UserModel } from '../../users/models/user.model'
+import type { FileTask } from '../models/file-task'
+
+export interface FileTaskQueueItem {
+ cacheKey: string
+ dto: any
+ method: string
+ space: SpaceEnv
+ task: FileTask
+ user: UserModel
+}
+
+export type FileTaskQueueStarter = (task: FileTaskQueueItem) => Promise<boolean>
+
+export interface FileTaskQueueEntry {
+ task: FileTaskQueueItem
+ startTask: FileTaskQueueStarter
+}
diff --git a/backend/src/applications/files/models/file-task.ts b/backend/src/applications/files/models/file-task.ts
index 809396c9..d38ae6c0 100644
--- a/backend/src/applications/files/models/file-task.ts
+++ b/backend/src/applications/files/models/file-task.ts
@@ -4,7 +4,8 @@ export enum FileTaskStatus {
PENDING,
SUCCESS,
ERROR,
- CANCELLED
+ CANCELLED,
+ QUEUED
}
export interface FileTaskProps {
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index acc93432..44b10dc8 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -4,6 +4,7 @@ import { isNotNull, sql } from 'drizzle-orm'
import { unionAll } from 'drizzle-orm/mysql-core'
import fs from 'node:fs/promises'
import path from 'node:path'
+import { currentTimeStamp } from '../../../common/shared'
import { Cache } from '../../../infrastructure/cache/cache.service'
import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'
import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'
@@ -11,7 +12,7 @@ import { getTablesWithFileIdColumn } from '../../../infrastructure/database/util
import { USER_PATH, USER_ROLE } from '../../users/constants/user'
import { UserModel } from '../../users/models/user.model'
import { users } from '../../users/schemas/users.schema'
-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX } from '../constants/cache'
+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_USER_PREFIX } from '../constants/cache'
import { FileTask, FileTaskStatus } from '../models/file-task'
import { filesRecents } from '../schemas/files-recents.schema'
import { files } from '../schemas/files.schema'
@@ -260,6 +261,7 @@ export class FilesScheduler {
try {
let nb = 0
let nbCancellationRequests = 0
+ let nbUserTaskCounters = 0
const keys = await this.cache.keys(`${CACHE_TASK_PREFIX}-*`)
for (const key of keys) {
if (key.startsWith(`${CACHE_TASK_CANCEL_PREFIX}-`)) {
@@ -267,20 +269,33 @@ export class FilesScheduler {
nbCancellationRequests++
continue
}
+ if (key.startsWith(`${CACHE_TASK_USER_PREFIX}-`)) {
+ await this.cache.del(key)
+ nbUserTaskCounters++
+ continue
+ }
const task = await this.cache.get(key)
- if (task && task.status === FileTaskStatus.PENDING) {
+ if (task && this.isInterruptedTaskStatus(task.status)) {
task.status = FileTaskStatus.ERROR
task.result = 'Interrupted'
+ task.endedAt = currentTimeStamp(null, true)
nb++
this.cache.set(key, task).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))
}
}
- this.logger.log({ tag: this.cleanupInterruptedTasks.name, msg: `${nb} tasks cleaned, ${nbCancellationRequests} cancellation requests cleared` })
+ this.logger.log({
+ tag: this.cleanupInterruptedTasks.name,
+ msg: `${nb} tasks cleaned, ${nbCancellationRequests} cancellation requests cleared, ${nbUserTaskCounters} user task counters cleared`
+ })
} catch (e) {
this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` })
}
}
+ private isInterruptedTaskStatus(status: FileTaskStatus): boolean {
+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
+ }
+
private async resetContentIndexingState(): Promise<void> {
await this.filesContentIndexer.resetIndexingRuntimeState()
this.logger.log({ tag: this.resetContentIndexingState.name, msg: `done` })
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
index 6d848832..420749c9 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
@@ -6,10 +6,11 @@ import * as filesUtils from '../utils/files'
import { FileTaskEvent } from '../events/file-events'
import { SendFile } from '../utils/send-file'
import { FILE_OPERATION } from '../constants/operations'
-import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
+import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
import { FileTaskStatus } from '../models/file-task'
import { FilesMethods } from './files-methods.service'
import { FilesTasksManager } from './files-tasks-manager.service'
+import { FilesTasksQueue } from './files-tasks-queue.service'
import { Mock } from 'vitest'
describe(FilesTasksManager.name, () => {
@@ -24,6 +25,7 @@ describe(FilesTasksManager.name, () => {
get: Mock
keys: Mock
mget: Mock
+ increment: Mock
del: Mock
}
@@ -47,6 +49,11 @@ describe(FilesTasksManager.name, () => {
get: vi.fn(async (key: string) => cacheStore.get(key)),
keys: vi.fn(async (pattern: string) => [...cacheStore.keys()].filter((k) => createPatternRegex(pattern).test(k))),
mget: vi.fn(async (keys: string[]) => keys.map((k) => cacheStore.get(k)).filter((v) => v !== undefined)),
+ increment: vi.fn(async (key: string, amount = 1) => {
+ const value = (Number(cacheStore.get(key)) || 0) + amount
+ cacheStore.set(key, value)
+ return value
+ }),
del: vi.fn(async (key: string) => cacheStore.delete(key))
}
filesMethods = {
@@ -55,6 +62,7 @@ describe(FilesTasksManager.name, () => {
module = await Test.createTestingModule({
providers: [
FilesTasksManager,
+ FilesTasksQueue,
{
provide: Cache,
useValue: cache
@@ -97,6 +105,7 @@ describe(FilesTasksManager.name, () => {
expect(FilesTasksManager.getCacheKey(10, 'task-1')).toBe(`${CACHE_TASK_PREFIX}-10-task-1`)
expect(FilesTasksManager.getCacheKey(10)).toBe(`${CACHE_TASK_PREFIX}-10-*`)
expect(FilesTasksManager.getCancellationCacheKey(10, 'task-1')).toBe(`${CACHE_TASK_CANCEL_PREFIX}-10-task-1`)
+ expect(FilesTasksQueue.getUserRunningTasksCacheKey(10)).toBe(`${CACHE_TASK_USER_PREFIX}-10`)
})
it('should request cancellation through cache for a pending supported task', async () => {
@@ -283,6 +292,89 @@ describe(FilesTasksManager.name, () => {
expect(storedTask.result).toBe('operation failed')
})
+ it('should queue tasks when a user already has three running tasks', async () => {
+ vi.spyOn(crypto, 'randomUUID')
+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')
+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')
+ filesMethods.doWork.mockImplementation(() => new Promise(() => undefined))
+ const user = { id: 7 } as any
+
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')
+
+ expect(filesMethods.doWork).toHaveBeenCalledTimes(3)
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`).status).toBe(FileTaskStatus.PENDING)
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`).status).toBe(FileTaskStatus.QUEUED)
+ expect(cacheStore.get(`${CACHE_TASK_USER_PREFIX}-7`)).toBe(3)
+ })
+
+ it('should start the next queued task when a running task completes', async () => {
+ const resolveWork: (() => void)[] = []
+ vi.spyOn(crypto, 'randomUUID')
+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')
+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')
+ filesMethods.doWork.mockImplementation(() => new Promise<void>((resolve) => resolveWork.push(resolve)))
+ const user = { id: 7 } as any
+
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.COPY, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')
+
+ resolveWork[0]()
+ await flushPromises()
+
+ expect(filesMethods.doWork).toHaveBeenCalledTimes(4)
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-11111111-1111-4111-8111-111111111111`).status).toBe(FileTaskStatus.SUCCESS)
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`).status).toBe(FileTaskStatus.PENDING)
+ expect(cacheStore.get(`${CACHE_TASK_USER_PREFIX}-7`)).toBe(3)
+ })
+
+ it('should cancel a queued cancellable task without starting it', async () => {
+ vi.spyOn(crypto, 'randomUUID')
+ .mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
+ .mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
+ .mockReturnValueOnce('33333333-3333-4333-8333-333333333333')
+ .mockReturnValueOnce('44444444-4444-4444-8444-444444444444')
+ filesMethods.doWork.mockImplementation(() => new Promise(() => undefined))
+ const user = { id: 7 } as any
+
+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-1.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-2.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-3.txt' } as any, null, 'doWork')
+ await filesTasksManager.createTask(FILE_OPERATION.DOWNLOAD, user, { url: 'files/personal/document-4.txt' } as any, null, 'doWork')
+
+ await filesTasksManager.cancelTask(7, '44444444-4444-4444-8444-444444444444')
+
+ const storedTask = cacheStore.get(`${CACHE_TASK_PREFIX}-7-44444444-4444-4444-8444-444444444444`)
+ expect(filesMethods.doWork).toHaveBeenCalledTimes(3)
+ expect(storedTask.status).toBe(FileTaskStatus.CANCELLED)
+ expect(storedTask.result).toBe('Cancelled')
+ })
+
+ it('should cancel a cached queued cancellable task even when it is not in the local queue', async () => {
+ cacheStore.set(`${CACHE_TASK_PREFIX}-8-task-queued`, {
+ id: 'task-queued',
+ type: FILE_OPERATION.DOWNLOAD,
+ status: FileTaskStatus.QUEUED,
+ props: {}
+ })
+
+ await filesTasksManager.cancelTask(8, 'task-queued')
+
+ const storedTask = cacheStore.get(`${CACHE_TASK_PREFIX}-8-task-queued`)
+ expect(storedTask.status).toBe(FileTaskStatus.CANCELLED)
+ expect(storedTask.result).toBe('Cancelled')
+ expect(storedTask.endedAt).toBeDefined()
+ expect(cache.set).not.toHaveBeenCalledWith(`${CACHE_TASK_CANCEL_PREFIX}-8-task-queued`, true, CACHE_TASK_TTL)
+ })
+
it('should return one task by id and throw when it does not exist', async () => {
cacheStore.set(`${CACHE_TASK_PREFIX}-11-task-a`, { id: 'task-a' })
@@ -319,7 +411,7 @@ describe(FilesTasksManager.name, () => {
expect(cache.del).toHaveBeenCalledWith(`${CACHE_TASK_PREFIX}-22-task-ok`)
})
- it('should ignore pending tasks when deleting all tasks', async () => {
+ it('should ignore active tasks when deleting all tasks', async () => {
const removeFilesSpy = vi.spyOn(filesUtils, 'removeFiles').mockResolvedValue(undefined)
const stopWatchSpy = vi.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValue(undefined)
const user = { id: 31, tasksPath: '/tmp/tasks' } as any
@@ -328,6 +420,11 @@ describe(FilesTasksManager.name, () => {
status: FileTaskStatus.PENDING,
props: { compressInDirectory: false }
})
+ cacheStore.set(`${CACHE_TASK_PREFIX}-31-task-queued`, {
+ id: 'task-queued',
+ status: FileTaskStatus.QUEUED,
+ props: { compressInDirectory: false }
+ })
cacheStore.set(`${CACHE_TASK_PREFIX}-31-task-done`, {
id: 'task-done',
name: 'done.tar',
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.ts
index c88e849c..dbd9890c 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts
@@ -9,11 +9,13 @@ import { SpaceEnv } from '../../spaces/models/space-env.model'
import { UserModel } from '../../users/models/user.model'
import { CACHE_TASK_CANCEL_PREFIX, CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
import { FILE_OPERATION } from '../constants/operations'
+import { FileTaskEvent } from '../events/file-events'
+import type { FileTaskQueueItem } from '../interfaces/file-task-queue.interface'
import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'
import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'
import { SendFile } from '../utils/send-file'
import { FilesMethods } from './files-methods.service'
-import { FileTaskEvent } from '../events/file-events'
+import { FilesTasksQueue } from './files-tasks-queue.service'
@Injectable()
export class FilesTasksManager implements OnModuleDestroy {
@@ -25,7 +27,8 @@ export class FilesTasksManager implements OnModuleDestroy {
constructor(
private readonly cache: Cache,
- private readonly filesMethods: FilesMethods
+ private readonly filesMethods: FilesMethods,
+ private readonly filesTasksQueue: FilesTasksQueue
) {
FileTaskEvent.on('startWatch', this.onStartWatch)
}
@@ -52,23 +55,9 @@ export class FilesTasksManager implements OnModuleDestroy {
const taskId: string = crypto.randomUUID()
const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)
const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url))
- await this.storeTask(cacheKey, newTask)
- space.task = { cacheKey: cacheKey, props: {} }
- const controller = this.watchCancellation(type, user.id, taskId, cacheKey)
- const taskPromise = controller ? this.filesMethods[method](user, space, dto, controller.signal) : this.filesMethods[method](user, space, dto)
- taskPromise
- .then((data: any) => {
- this.logger.debug({ tag: this.createTask.name, msg: `${newTask.name} : ${method} done` })
- this.setTaskDone(cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) => this.logger.error({ tag: this.createTask.name, msg: `${e}` }))
- })
- .catch((e: HttpException | any) => {
- this.logger.error({ tag: this.createTask.name, msg: `${newTask.name} : ${method} : ${e}` })
- this.setTaskDone(
- cacheKey,
- controller?.signal.aborted ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,
- controller?.signal.aborted ? 'Cancelled' : e.message
- ).catch((e: Error) => this.logger.error({ tag: this.createTask.name, msg: `${e}` }))
- })
+ await this.storeTask(cacheKey, newTask, FileTaskStatus.QUEUED)
+ space.task = { cacheKey, props: {} }
+ await this.filesTasksQueue.enqueue(user.id, { cacheKey, dto, method, space, task: newTask, user }, (task) => this.startQueuedTask(task))
return newTask
}
@@ -86,16 +75,11 @@ export class FilesTasksManager implements OnModuleDestroy {
async deleteTasks(user: UserModel, taskId?: string): Promise<void> {
const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)
- let keys: string[]
- if (taskId) {
- keys = [cacheKey]
- } else {
- keys = await this.cache.keys(cacheKey)
- }
+ const keys: string[] = taskId ? [cacheKey] : await this.cache.keys(cacheKey)
if (!keys.length) return
for (const key of keys) {
const task: FileTask = await this.cache.get(key)
- if (!task || task.status === FileTaskStatus.PENDING) continue
+ if (!task || this.isActiveStatus(task.status)) continue
if (task.props.compressInDirectory === false) {
// delete task file
const rPath = path.join(user.tasksPath, task.name)
@@ -114,9 +98,14 @@ export class FilesTasksManager implements OnModuleDestroy {
if (!task) {
throw new HttpException('Task not found', HttpStatus.NOT_FOUND)
}
- if (task.status !== FileTaskStatus.PENDING || !this.canCancel(task.type)) {
+ if (!this.isActiveStatus(task.status) || !this.canCancel(task.type)) {
throw new HttpException('Not applicable', HttpStatus.BAD_REQUEST)
}
+ if (task.status === FileTaskStatus.QUEUED) {
+ this.filesTasksQueue.remove(userId, taskId)
+ await this.setTaskDone(cacheKey, FileTaskStatus.CANCELLED, 'Cancelled')
+ return
+ }
const isStored = await this.cache.set(FilesTasksManager.getCancellationCacheKey(userId, taskId), true, CACHE_TASK_TTL)
if (!isStored) {
throw new HttpException('Unable to cancel task', HttpStatus.INTERNAL_SERVER_ERROR)
@@ -139,9 +128,9 @@ export class FilesTasksManager implements OnModuleDestroy {
return await sendFile.stream(req, res)
}
- private async storeTask(cacheKey: string, task: FileTask) {
+ private async storeTask(cacheKey: string, task: FileTask, status = FileTaskStatus.PENDING) {
task.startedAt = currentTimeStamp(null, true)
- task.status = FileTaskStatus.PENDING
+ task.status = status
try {
await this.cache.set(cacheKey, task, CACHE_TASK_TTL)
} catch (e) {
@@ -170,6 +159,52 @@ export class FilesTasksManager implements OnModuleDestroy {
.catch((e: Error) => this.logger.error({ tag: this.setTaskDone.name, msg: `${e}` }))
}
+ private async startQueuedTask(task: FileTaskQueueItem): Promise<boolean> {
+ const storedTask: FileTask = await this.cache.get(task.cacheKey)
+ if (!storedTask || storedTask.status !== FileTaskStatus.QUEUED) return false
+ task.task.status = FileTaskStatus.PENDING
+ task.task.startedAt = currentTimeStamp(null, true)
+ task.space.task = { cacheKey: task.cacheKey, props: task.task.props }
+ await this.cache.set(task.cacheKey, task.task, CACHE_TASK_TTL)
+ this.runTask(task)
+ return true
+ }
+
+ private runTask(task: FileTaskQueueItem): void {
+ const controller = this.watchCancellation(task.task.type, task.user.id, task.task.id, task.cacheKey)
+ const taskPromise = controller
+ ? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)
+ : this.filesMethods[task.method](task.user, task.space, task.dto)
+ taskPromise
+ .then((data: any) => {
+ this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })
+ this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data).catch((e: Error) =>
+ this.logger.error({ tag: this.runTask.name, msg: `${e}` })
+ )
+ })
+ .catch((e: HttpException | any) => {
+ this.logger.error({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} : ${e}` })
+ this.completeTask(
+ task.user.id,
+ task.cacheKey,
+ controller?.signal.aborted ? FileTaskStatus.CANCELLED : FileTaskStatus.ERROR,
+ controller?.signal.aborted ? 'Cancelled' : e.message
+ ).catch((e: Error) => this.logger.error({ tag: this.runTask.name, msg: `${e}` }))
+ })
+ }
+
+ private async completeTask(userId: number, cacheKey: string, status: FileTaskStatus, result: any): Promise<void> {
+ try {
+ await this.setTaskDone(cacheKey, status, result)
+ } finally {
+ await this.filesTasksQueue.releaseAndDrain(userId)
+ }
+ }
+
+ private isActiveStatus(status: FileTaskStatus): boolean {
+ return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
+ }
+
private async updateTask(cacheKey: string, props?: FileTaskProps, task?: Partial<FileTask>): Promise<void> {
let ftask: FileTask = await this.cache.get(cacheKey)
if (ftask) {
diff --git a/backend/src/applications/files/services/files-tasks-queue.service.spec.ts b/backend/src/applications/files/services/files-tasks-queue.service.spec.ts
new file mode 100644
index 00000000..3f45541d
--- /dev/null
+++ b/backend/src/applications/files/services/files-tasks-queue.service.spec.ts
@@ -0,0 +1,137 @@
+import { Mock } from 'vitest'
+import { Cache } from '../../../infrastructure/cache/cache.service'
+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
+import type { FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'
+import { FilesTasksQueue } from './files-tasks-queue.service'
+
+describe(FilesTasksQueue.name, () => {
+ const userId = 42
+ const counterKey = `${CACHE_TASK_USER_PREFIX}-${userId}`
+ let counters: Map<string, number>
+ let increment: Mock
+ let queue: FilesTasksQueue
+
+ const createTask = (id: string): FileTaskQueueItem =>
+ ({
+ cacheKey: `ftask-${userId}-${id}`,
+ task: { id }
+ }) as FileTaskQueueItem
+
+ beforeEach(() => {
+ counters = new Map<string, number>()
+ increment = vi.fn(async (key: string, amount = 1, _ttl?: number, minimum?: number) => {
+ const value = Math.max((counters.get(key) ?? 0) + amount, minimum ?? Number.NEGATIVE_INFINITY)
+ counters.set(key, value)
+ return value
+ })
+ queue = new FilesTasksQueue({ increment } as unknown as Cache)
+ })
+
+ afterEach(() => {
+ queue.stop()
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('should build the shared running tasks cache key', () => {
+ expect(FilesTasksQueue.getUserRunningTasksCacheKey(userId)).toBe(counterKey)
+ })
+
+ it('should start at most three tasks per user and queue the next one', async () => {
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)
+ const tasks = ['task-1', 'task-2', 'task-3', 'task-4'].map(createTask)
+
+ for (const task of tasks) {
+ await queue.enqueue(userId, task, startTask)
+ }
+
+ expect(startTask).toHaveBeenCalledTimes(3)
+ expect(startTask.mock.calls.map(([task]) => task.task.id)).toEqual(['task-1', 'task-2', 'task-3'])
+ expect(counters.get(counterKey)).toBe(3)
+ })
+
+ it('should apply the running task limit independently for each user', async () => {
+ const secondUserId = 84
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)
+
+ for (const currentUserId of [userId, secondUserId]) {
+ for (const id of ['task-1', 'task-2', 'task-3']) {
+ await queue.enqueue(currentUserId, createTask(`${currentUserId}-${id}`), startTask)
+ }
+ }
+
+ expect(startTask).toHaveBeenCalledTimes(6)
+ expect(counters.get(counterKey)).toBe(3)
+ expect(counters.get(`${CACHE_TASK_USER_PREFIX}-${secondUserId}`)).toBe(3)
+ })
+
+ it('should start the next queued task after releasing a slot', async () => {
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)
+
+ for (const id of ['task-1', 'task-2', 'task-3', 'task-4']) {
+ await queue.enqueue(userId, createTask(id), startTask)
+ }
+ await queue.releaseAndDrain(userId)
+
+ expect(startTask).toHaveBeenCalledTimes(4)
+ expect(startTask).toHaveBeenLastCalledWith(expect.objectContaining({ task: expect.objectContaining({ id: 'task-4' }) }))
+ expect(counters.get(counterKey)).toBe(3)
+ })
+
+ it('should remove a queued task without starting it', async () => {
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)
+
+ for (const id of ['task-1', 'task-2', 'task-3', 'task-4']) {
+ await queue.enqueue(userId, createTask(id), startTask)
+ }
+ queue.remove(userId, 'task-4')
+ await queue.releaseAndDrain(userId)
+
+ expect(startTask).toHaveBeenCalledTimes(3)
+ expect(counters.get(counterKey)).toBe(2)
+ })
+
+ it('should release the claimed slot when a task cannot start', async () => {
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(false)
+
+ await queue.enqueue(userId, createTask('task-1'), startTask)
+
+ expect(startTask).toHaveBeenCalledOnce()
+ expect(counters.get(counterKey)).toBe(0)
+ expect(increment).toHaveBeenLastCalledWith(counterKey, -1, CACHE_TASK_TTL, 0)
+ })
+
+ it('should release the claimed slot when the task starter throws', async () => {
+ const error = new Error('start failed')
+ const startTask = vi.fn<FileTaskQueueStarter>().mockRejectedValue(error)
+ const loggerSpy = vi.spyOn((queue as any).logger, 'error').mockImplementation(() => undefined)
+
+ await queue.enqueue(userId, createTask('task-1'), startTask)
+
+ expect(loggerSpy).toHaveBeenCalledWith({ tag: 'drain', msg: `${error}` })
+ expect(counters.get(counterKey)).toBe(0)
+ })
+
+ it('should never decrement the shared counter below zero', async () => {
+ await queue.releaseAndDrain(userId)
+ await queue.releaseAndDrain(userId)
+
+ expect(counters.get(counterKey)).toBe(0)
+ expect(increment).toHaveBeenLastCalledWith(counterKey, -1, CACHE_TASK_TTL, 0)
+ })
+
+ it('should periodically retry queued tasks when a shared slot becomes available', async () => {
+ vi.useFakeTimers()
+ counters.set(counterKey, 3)
+ const startTask = vi.fn<FileTaskQueueStarter>().mockResolvedValue(true)
+
+ await queue.enqueue(userId, createTask('task-1'), startTask)
+ expect(startTask).not.toHaveBeenCalled()
+
+ counters.set(counterKey, 2)
+ await vi.advanceTimersByTimeAsync(1000)
+
+ expect(startTask).toHaveBeenCalledOnce()
+ expect(counters.get(counterKey)).toBe(3)
+ })
+})
diff --git a/backend/src/applications/files/services/files-tasks-queue.service.ts b/backend/src/applications/files/services/files-tasks-queue.service.ts
new file mode 100644
index 00000000..75f02f87
--- /dev/null
+++ b/backend/src/applications/files/services/files-tasks-queue.service.ts
@@ -0,0 +1,117 @@
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'
+import { Cache } from '../../../infrastructure/cache/cache.service'
+import { CACHE_TASK_TTL, CACHE_TASK_USER_PREFIX } from '../constants/cache'
+import type { FileTaskQueueEntry, FileTaskQueueItem, FileTaskQueueStarter } from '../interfaces/file-task-queue.interface'
+
+@Injectable()
+export class FilesTasksQueue implements OnModuleDestroy {
+ private readonly logger = new Logger(FilesTasksQueue.name)
+ private readonly maxRunningTasksPerUser = 3
+ private readonly drainInterval = 1000
+ private drainingUsers = new Set<number>()
+ private queuedTasks: Record<number, FileTaskQueueEntry[]> = {}
+ private queuedTasksDrainInterval: NodeJS.Timeout | undefined
+
+ constructor(private readonly cache: Cache) {}
+
+ static getUserRunningTasksCacheKey(userId: number): string {
+ return `${CACHE_TASK_USER_PREFIX}-${userId}`
+ }
+
+ onModuleDestroy(): void {
+ this.stop()
+ }
+
+ async enqueue(userId: number, task: FileTaskQueueItem, startTask: FileTaskQueueStarter): Promise<void> {
+ ;(this.queuedTasks[userId] ??= []).push({ task, startTask })
+ this.startQueueDrainInterval()
+ await this.drain(userId)
+ }
+
+ remove(userId: number, taskId: string): void {
+ const queue = this.queuedTasks[userId]
+ if (!queue) return
+ const taskIndex = queue.findIndex((entry: FileTaskQueueEntry) => entry.task.task.id === taskId)
+ if (taskIndex !== -1) {
+ queue.splice(taskIndex, 1)
+ }
+ if (!queue.length) {
+ delete this.queuedTasks[userId]
+ this.stopQueueDrainIntervalIfIdle()
+ }
+ }
+
+ async releaseAndDrain(userId: number): Promise<void> {
+ await this.releaseUserRunningTaskSlot(userId)
+ await this.drain(userId)
+ }
+
+ stop(): void {
+ if (!this.queuedTasksDrainInterval) return
+ clearInterval(this.queuedTasksDrainInterval)
+ this.queuedTasksDrainInterval = undefined
+ }
+
+ private async drain(userId: number): Promise<void> {
+ if (this.drainingUsers.has(userId)) return
+ this.drainingUsers.add(userId)
+ try {
+ const queue = this.queuedTasks[userId]
+ if (!queue?.length) {
+ this.stopQueueDrainIntervalIfIdle()
+ return
+ }
+ while (queue.length) {
+ if (!(await this.claimUserRunningTaskSlot(userId))) break
+ const entry = queue.shift()
+ try {
+ if (entry && (await entry.startTask(entry.task))) continue
+ } catch (e) {
+ this.logger.error({ tag: this.drain.name, msg: `${e}` })
+ }
+ await this.releaseUserRunningTaskSlot(userId)
+ }
+ if (!queue.length) {
+ delete this.queuedTasks[userId]
+ this.stopQueueDrainIntervalIfIdle()
+ }
+ } finally {
+ this.drainingUsers.delete(userId)
+ }
+ }
+
+ private async claimUserRunningTaskSlot(userId: number): Promise<boolean> {
+ const cacheKey = FilesTasksQueue.getUserRunningTasksCacheKey(userId)
+ const runningTasks = await this.cache.increment(cacheKey, 1, CACHE_TASK_TTL)
+ if (runningTasks <= this.maxRunningTasksPerUser) {
+ return true
+ }
+ await this.releaseUserRunningTaskSlot(userId)
+ return false
+ }
+
+ private async releaseUserRunningTaskSlot(userId: number): Promise<void> {
+ const cacheKey = FilesTasksQueue.getUserRunningTasksCacheKey(userId)
+ await this.cache.increment(cacheKey, -1, CACHE_TASK_TTL, 0)
+ }
+
+ private startQueueDrainInterval(): void {
+ if (this.queuedTasksDrainInterval) return
+ this.queuedTasksDrainInterval = setInterval(() => {
+ this.drainAll().catch((e: Error) => this.logger.error({ tag: this.drainAll.name, msg: `${e}` }))
+ }, this.drainInterval)
+ this.queuedTasksDrainInterval.unref()
+ }
+
+ private stopQueueDrainIntervalIfIdle(): void {
+ if (Object.keys(this.queuedTasks).length) return
+ this.stop()
+ }
+
+ private async drainAll(): Promise<void> {
+ for (const userId of Object.keys(this.queuedTasks)) {
+ await this.drain(Number(userId))
+ }
+ this.stopQueueDrainIntervalIfIdle()
+ }
+}