Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): clean orphan task files
johaven0d4b306b87e1
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 419a405e..7f690045 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -54,6 +54,7 @@ describe(FilesManager.name, () => {}const user = { id: 7, login: 'john', tmpPath: '/data/users/john/tmp', tasksPath: '/data/users/john/tmp/tasks' } as any+ const taskPath = (cacheKey: string, name: string): string => path.join(user.tasksPath, `${taskUtils.taskTemporaryPrefix(cacheKey)}${name}`)const makeSpace = (overrides: Record<string, any> = {}) =>({@@ -195,9 +196,9 @@ describe(FilesManager.name, () => {vi.spyOn(filesUtils, 'dirSize').mockResolvedValue([123, {}] as any)vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/unique-path.txt')vi.spyOn(filesUtils, 'uniqueDatedFilePath').mockResolvedValue({ isDir: false, path: '/trash/file-2026.txt' })- vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue('/data/users/john/tmp/tasks/.task-d-archive')+ vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue(taskPath('task-d', 'archive'))vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>- path.join(parentPath, `.${cacheKey}-${path.basename(name)}`)+ path.join(parentPath, `${taskUtils.taskTemporaryPrefix(cacheKey)}${path.basename(name)}`))vi.spyOn(tarUtils, 'createTar').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'getMimeType').mockReturnValue('image-png')@@ -1038,7 +1039,7 @@ describe(FilesManager.name, () => {expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()expect(filesUtils.writeFromStream).toHaveBeenCalledWith(- '/data/users/john/tmp/tasks/.task-1-download.txt',+ taskPath('task-1', 'download.txt'),expect.anything(),0,55,@@ -1046,7 +1047,7 @@ describe(FilesManager.name, () => {expect.any(Function))expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-1', 'download.txt'), '/tmp/download.txt')expect(filesLockManager.create).toHaveBeenCalledWith(user,expect.objectContaining({ path: 'download.txt' }),@@ -1128,19 +1129,12 @@ describe(FilesManager.name, () => {await service.compress(user, space, dto)- expect(tarSpy).toHaveBeenCalledWith(- '/data/users/john/tmp/tasks/.task-c-archive.tgz',- dto.files,- true,- undefined,- expect.any(Function),- undefined- )+ expect(tarSpy).toHaveBeenCalledWith(taskPath('task-c', 'archive.tgz'), dto.files, true, undefined, expect.any(Function), undefined)expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tgz')expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tgz')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()expect(space.task.props.size).toBe(Buffer.byteLength('content'))- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', '/tmp/archive.tgz')+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-c', 'archive.tgz'), '/tmp/archive.tgz')})it('should allow archive export from trash when compressInDirectory is false', async () => {@@ -1277,12 +1271,12 @@ describe(FilesManager.name, () => {expect(filesTasksTransfer.createExtractionProgressHandler).toHaveBeenCalledWith(space)expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip',- '/data/users/john/tmp/tasks/.task-d-archive',+ taskPath('task-d', 'archive'),undefined,undefined,expect.any(Function))- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-d', 'archive'), '/data/users/john/files/archive')expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/data/users/john/files/archive')expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')})diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.tsindex 0c8395ce..3be082a1 100644--- a/backend/src/applications/files/services/files-scheduler.service.ts+++ b/backend/src/applications/files/services/files-scheduler.service.ts@@ -45,6 +45,7 @@ export class FilesScheduler {try {await this.resetContentIndexingState()await this.cleanupInterruptedTasks()+ await this.cleanupUserTmpFiles()await this.clearRecentFiles()await this.updateQuotas()await this.cleanupTrashFiles()@@ -215,22 +216,27 @@ export class FilesScheduler {private async cleanupUserTaskFiles(userId: number, userTasksPath: string, expiration: number): Promise<void> {try {+ // Snapshot the directory first so files created by tasks registered during cache lookup are left for the next pass.+ const fileNames = await fs.readdir(userTasksPath)const keys = await this.cache.keys(FilesTasksManager.getCacheKey(userId))const tasks: (FileTask | null | undefined)[] = keys.length ? await this.cache.mget(keys) : []const protectedFiles = new Set<string>()const protectedPrefixes: string[] = []+ let hasActiveTasks = falsefor (const task of tasks) {- if (!task || !isActiveTaskStatus(task.status)) continue+ if (!task) continue+ // Exported archives remain downloadable while their completed task is still cached.+ if (task.props.compressInDirectory === false) protectedFiles.add(task.name)+ if (!isActiveTaskStatus(task.status)) continue+ hasActiveTasks = true// Active task staging files must survive tmp cleanup; the transfer may still publish or rollback them.// QUEUED tasks are included because they may start after this cache snapshot.protectedPrefixes.push(taskTemporaryPrefix(FilesTasksManager.getCacheKey(userId, task.id)))- // Exported archives are final results stored beside staging entries and do not use the task prefix.- if (task.props.compressInDirectory === false) protectedFiles.add(task.name)}- for (const fileName of await fs.readdir(userTasksPath)) {+ for (const fileName of fileNames) {if (protectedFiles.has(fileName) || protectedPrefixes.some((prefix: string) => fileName.startsWith(prefix))) continue- // The age guard protects tasks created after the cache snapshot from being removed by this cleanup pass.- await this.removeTmpFile(path.join(userTasksPath, fileName), expiration)+ // An active task may have just published an unprefixed result before updating its cached name.+ await this.removeTmpFile(path.join(userTasksPath, fileName), hasActiveTasks ? expiration : undefined)}} catch (e) {this.logger.error({ tag: this.cleanupUserTaskFiles.name, msg: `unable to browse ${userTasksPath} : ${e}` })@@ -293,7 +299,7 @@ export class FilesScheduler {task.result = 'Interrupted'task.endedAt = currentTimeStamp(null, true)nb++- this.cache.set(key, task, CACHE_TASK_TTL).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))+ await this.cache.set(key, task, CACHE_TASK_TTL)}}this.logger.log({diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.tsindex e7e06f62..eec8f82c 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts@@ -2,6 +2,7 @@ import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/pimport os from 'node:os'import path from 'node:path'import * as filesUtils from '../../utils/files'+import { taskTemporaryPrefix } from '../../utils/tasks'import { FilesTasksTransfer } from './files-tasks-transfer.service'import { SourceCleanupError } from '../../models/file-error'@@ -74,7 +75,7 @@ describe(FilesTasksTransfer.name, () => {it('publishes a copied file only after the commit hook', async () => {const signal = new AbortController().signal- const temporaryPath = path.join(stagingDir, `.${cacheKey}-destination.txt`)+ const temporaryPath = path.join(stagingDir, `${taskTemporaryPrefix(cacheKey)}destination.txt`)let transferredBytes = 0await writeFile(srcPath, 'content')@@ -94,7 +95,7 @@ describe(FilesTasksTransfer.name, () => {expect(transferredBytes).toBe(Buffer.byteLength('content'))expect(path.dirname(temporaryPath)).toBe(stagingDir)- expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)+ expect(path.basename(temporaryPath)).toBe(`${taskTemporaryPrefix(cacheKey)}destination.txt`)await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })})diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.tsindex dbd566c8..ed74af87 100644--- a/backend/src/applications/files/utils/tasks.spec.ts+++ b/backend/src/applications/files/utils/tasks.spec.ts@@ -2,7 +2,7 @@ import fs, { mkdtemp, rm, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { FILE_OPERATION } from '../constants/operations'-import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath } from './tasks'+import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath, taskTemporaryPrefix } from './tasks'describe('file task utilities', () => {const cacheKey = 'ftask-7-task-id'@@ -45,8 +45,8 @@ describe('file task utilities', () => {const filePath = taskTemporaryPath(tmpDir, cacheKey, '/destination/report.txt')const directoryPath = await createTaskTemporaryDir(tmpDir, cacheKey, '/destination/archive')- expect(path.basename(filePath)).toBe(`.${cacheKey}-report.txt`)- expect(path.basename(directoryPath)).toBe(`.${cacheKey}-archive`)+ expect(path.basename(filePath)).toBe(`${taskTemporaryPrefix(cacheKey)}report.txt`)+ expect(path.basename(directoryPath)).toBe(`${taskTemporaryPrefix(cacheKey)}archive`)expect((await fs.stat(directoryPath)).isDirectory()).toBe(true)})})diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.tsindex 03013c0d..44343e68 100644--- a/backend/src/applications/files/utils/tasks.ts+++ b/backend/src/applications/files/utils/tasks.ts@@ -6,7 +6,7 @@ import { isCrossDevice, walkDir } from './files'import { FileTaskProps, FileTaskStatus } from '../models/file-task'export function taskTemporaryPrefix(cacheKey: string): string {- return `.${cacheKey}-`+ return `~${cacheKey}-`}export function taskTemporaryPath(parentPath: string, cacheKey: string, name: string): string {