Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): clean orphan task files

johaven0d4b306b87e1
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 419a405e..7f690045 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -54,6 +54,7 @@ describe(FilesManager.name, () => {
}
const user = { id: 7, login: 'john', tmpPath: '/data/users/john/tmp', tasksPath: '/data/users/john/tmp/tasks' } as any
+ const taskPath = (cacheKey: string, name: string): string => path.join(user.tasksPath, `${taskUtils.taskTemporaryPrefix(cacheKey)}${name}`)
const makeSpace = (overrides: Record<string, any> = {}) =>
({
@@ -195,9 +196,9 @@ describe(FilesManager.name, () => {
vi.spyOn(filesUtils, 'dirSize').mockResolvedValue([123, {}] as any)
vi.spyOn(filesUtils, 'uniqueFilePathFromDir').mockResolvedValue('/tmp/unique-path.txt')
vi.spyOn(filesUtils, 'uniqueDatedFilePath').mockResolvedValue({ isDir: false, path: '/trash/file-2026.txt' })
- vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue('/data/users/john/tmp/tasks/.task-d-archive')
+ vi.spyOn(taskUtils, 'createTaskTemporaryDir').mockResolvedValue(taskPath('task-d', 'archive'))
vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>
- path.join(parentPath, `.${cacheKey}-${path.basename(name)}`)
+ path.join(parentPath, `${taskUtils.taskTemporaryPrefix(cacheKey)}${path.basename(name)}`)
)
vi.spyOn(tarUtils, 'createTar').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'getMimeType').mockReturnValue('image-png')
@@ -1038,7 +1039,7 @@ describe(FilesManager.name, () => {
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
expect(filesUtils.writeFromStream).toHaveBeenCalledWith(
- '/data/users/john/tmp/tasks/.task-1-download.txt',
+ taskPath('task-1', 'download.txt'),
expect.anything(),
0,
55,
@@ -1046,7 +1047,7 @@ describe(FilesManager.name, () => {
expect.any(Function)
)
expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-1', 'download.txt'), '/tmp/download.txt')
expect(filesLockManager.create).toHaveBeenCalledWith(
user,
expect.objectContaining({ path: 'download.txt' }),
@@ -1128,19 +1129,12 @@ describe(FilesManager.name, () => {
await service.compress(user, space, dto)
- expect(tarSpy).toHaveBeenCalledWith(
- '/data/users/john/tmp/tasks/.task-c-archive.tgz',
- dto.files,
- true,
- undefined,
- expect.any(Function),
- undefined
- )
+ expect(tarSpy).toHaveBeenCalledWith(taskPath('task-c', 'archive.tgz'), dto.files, true, undefined, expect.any(Function), undefined)
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tgz')
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tgz')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
expect(space.task.props.size).toBe(Buffer.byteLength('content'))
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', '/tmp/archive.tgz')
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-c', 'archive.tgz'), '/tmp/archive.tgz')
})
it('should allow archive export from trash when compressInDirectory is false', async () => {
@@ -1277,12 +1271,12 @@ describe(FilesManager.name, () => {
expect(filesTasksTransfer.createExtractionProgressHandler).toHaveBeenCalledWith(space)
expect(unzipSpy).toHaveBeenCalledWith(
'/data/users/john/files/archive.zip',
- '/data/users/john/tmp/tasks/.task-d-archive',
+ taskPath('task-d', 'archive'),
undefined,
undefined,
expect.any(Function)
)
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith(taskPath('task-d', 'archive'), '/data/users/john/files/archive')
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/data/users/john/files/archive')
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
})
diff --git a/backend/src/applications/files/services/files-scheduler.service.ts b/backend/src/applications/files/services/files-scheduler.service.ts
index 0c8395ce..3be082a1 100644
--- a/backend/src/applications/files/services/files-scheduler.service.ts
+++ b/backend/src/applications/files/services/files-scheduler.service.ts
@@ -45,6 +45,7 @@ export class FilesScheduler {
try {
await this.resetContentIndexingState()
await this.cleanupInterruptedTasks()
+ await this.cleanupUserTmpFiles()
await this.clearRecentFiles()
await this.updateQuotas()
await this.cleanupTrashFiles()
@@ -215,22 +216,27 @@ export class FilesScheduler {
private async cleanupUserTaskFiles(userId: number, userTasksPath: string, expiration: number): Promise<void> {
try {
+ // Snapshot the directory first so files created by tasks registered during cache lookup are left for the next pass.
+ const fileNames = await fs.readdir(userTasksPath)
const keys = await this.cache.keys(FilesTasksManager.getCacheKey(userId))
const tasks: (FileTask | null | undefined)[] = keys.length ? await this.cache.mget(keys) : []
const protectedFiles = new Set<string>()
const protectedPrefixes: string[] = []
+ let hasActiveTasks = false
for (const task of tasks) {
- if (!task || !isActiveTaskStatus(task.status)) continue
+ if (!task) continue
+ // Exported archives remain downloadable while their completed task is still cached.
+ if (task.props.compressInDirectory === false) protectedFiles.add(task.name)
+ if (!isActiveTaskStatus(task.status)) continue
+ hasActiveTasks = true
// Active task staging files must survive tmp cleanup; the transfer may still publish or rollback them.
// QUEUED tasks are included because they may start after this cache snapshot.
protectedPrefixes.push(taskTemporaryPrefix(FilesTasksManager.getCacheKey(userId, task.id)))
- // Exported archives are final results stored beside staging entries and do not use the task prefix.
- if (task.props.compressInDirectory === false) protectedFiles.add(task.name)
}
- for (const fileName of await fs.readdir(userTasksPath)) {
+ for (const fileName of fileNames) {
if (protectedFiles.has(fileName) || protectedPrefixes.some((prefix: string) => fileName.startsWith(prefix))) continue
- // The age guard protects tasks created after the cache snapshot from being removed by this cleanup pass.
- await this.removeTmpFile(path.join(userTasksPath, fileName), expiration)
+ // An active task may have just published an unprefixed result before updating its cached name.
+ await this.removeTmpFile(path.join(userTasksPath, fileName), hasActiveTasks ? expiration : undefined)
}
} catch (e) {
this.logger.error({ tag: this.cleanupUserTaskFiles.name, msg: `unable to browse ${userTasksPath} : ${e}` })
@@ -293,7 +299,7 @@ export class FilesScheduler {
task.result = 'Interrupted'
task.endedAt = currentTimeStamp(null, true)
nb++
- this.cache.set(key, task, CACHE_TASK_TTL).catch((e: Error) => this.logger.error({ tag: this.cleanupInterruptedTasks.name, msg: `${e}` }))
+ await this.cache.set(key, task, CACHE_TASK_TTL)
}
}
this.logger.log({
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
index e7e06f62..eec8f82c 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
@@ -2,6 +2,7 @@ import fs, { access, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/p
import os from 'node:os'
import path from 'node:path'
import * as filesUtils from '../../utils/files'
+import { taskTemporaryPrefix } from '../../utils/tasks'
import { FilesTasksTransfer } from './files-tasks-transfer.service'
import { SourceCleanupError } from '../../models/file-error'
@@ -74,7 +75,7 @@ describe(FilesTasksTransfer.name, () => {
it('publishes a copied file only after the commit hook', async () => {
const signal = new AbortController().signal
- const temporaryPath = path.join(stagingDir, `.${cacheKey}-destination.txt`)
+ const temporaryPath = path.join(stagingDir, `${taskTemporaryPrefix(cacheKey)}destination.txt`)
let transferredBytes = 0
await writeFile(srcPath, 'content')
@@ -94,7 +95,7 @@ describe(FilesTasksTransfer.name, () => {
expect(transferredBytes).toBe(Buffer.byteLength('content'))
expect(path.dirname(temporaryPath)).toBe(stagingDir)
- expect(path.basename(temporaryPath)).toBe(`.${cacheKey}-destination.txt`)
+ expect(path.basename(temporaryPath)).toBe(`${taskTemporaryPrefix(cacheKey)}destination.txt`)
await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
await expect(access(temporaryPath)).rejects.toMatchObject({ code: 'ENOENT' })
})
diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.ts
index dbd566c8..ed74af87 100644
--- a/backend/src/applications/files/utils/tasks.spec.ts
+++ b/backend/src/applications/files/utils/tasks.spec.ts
@@ -2,7 +2,7 @@ import fs, { mkdtemp, rm, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { FILE_OPERATION } from '../constants/operations'
-import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath } from './tasks'
+import { createTaskTemporaryDir, isTaskCancellable, taskTemporaryPath, taskTemporaryPrefix } from './tasks'
describe('file task utilities', () => {
const cacheKey = 'ftask-7-task-id'
@@ -45,8 +45,8 @@ describe('file task utilities', () => {
const filePath = taskTemporaryPath(tmpDir, cacheKey, '/destination/report.txt')
const directoryPath = await createTaskTemporaryDir(tmpDir, cacheKey, '/destination/archive')
- expect(path.basename(filePath)).toBe(`.${cacheKey}-report.txt`)
- expect(path.basename(directoryPath)).toBe(`.${cacheKey}-archive`)
+ expect(path.basename(filePath)).toBe(`${taskTemporaryPrefix(cacheKey)}report.txt`)
+ expect(path.basename(directoryPath)).toBe(`${taskTemporaryPrefix(cacheKey)}archive`)
expect((await fs.stat(directoryPath)).isDirectory()).toBe(true)
})
})
diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.ts
index 03013c0d..44343e68 100644
--- a/backend/src/applications/files/utils/tasks.ts
+++ b/backend/src/applications/files/utils/tasks.ts
@@ -6,7 +6,7 @@ import { isCrossDevice, walkDir } from './files'
import { FileTaskProps, FileTaskStatus } from '../models/file-task'
export function taskTemporaryPrefix(cacheKey: string): string {
- return `.${cacheKey}-`
+ return `~${cacheKey}-`
}
export function taskTemporaryPath(parentPath: string, cacheKey: string, name: string): string {