Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): stage archive extraction in user temp directory

johaven06f142564a56
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.ts
index 42bb61dc..b90bab84 100644
--- a/backend/src/applications/files/interfaces/file-event.interface.ts
+++ b/backend/src/applications/files/interfaces/file-event.interface.ts
@@ -4,7 +4,7 @@ import type { ACTION } from '../../../common/constants'
import type { UserModel } from '../../users/models/user.model'
export interface FileTaskEventEmit {
- startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string]
+ startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string]
}
export interface FileEventType {
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index e0cf7011..c5c48e22 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -164,6 +164,7 @@ describe(FilesManager.name, () => {
jest.spyOn(filesUtils, 'isPathExists').mockResolvedValue(true)
jest.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)
jest.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)
+ jest.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')
jest.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)
jest.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')
jest.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)
@@ -977,41 +978,53 @@ describe(FilesManager.name, () => {
describe('decompress', () => {
it('should extract zip and release lock', async () => {
const space = makeSpace({ realPath: '/data/users/john/files/archive.zip', task: { cacheKey: 'task-d', props: {} } })
- ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)
+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
const unzipSpy = jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)
const taskEmitSpy = jest.spyOn(FileTaskEvent, 'emit')
await service.decompress(user, space)
- expect(filesUtils.makeDir).toHaveBeenCalledWith('/data/users/john/files/archive')
- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/files/archive')
- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DECOMPRESS, '/data/users/john/files/archive')
+ expect(filesUtils.makeTempDir).toHaveBeenCalledWith('/data/users/john/tmp', 'archive-extract-')
+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123')
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
+ expect(taskEmitSpy).toHaveBeenCalledWith(
+ 'startWatch',
+ space,
+ FILE_OPERATION.DECOMPRESS,
+ '/data/users/john/files/archive',
+ '/data/users/john/tmp/archive-extract-123'
+ )
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
})
it('should extract tar formats via extractTar', async () => {
const space = makeSpace({ realPath: '/data/users/john/files/archive.tar.gz' })
- ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)
+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
const untarSpy = jest.spyOn(untarUtils, 'extractTar').mockResolvedValueOnce(undefined)
await service.decompress(user, space)
- expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/files/archive', true)
+ expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
})
it('should remove partial extraction and skip add event on failure', async () => {
const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })
;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
const error = new Error('extraction failed')
jest.spyOn(unzipUtils, 'extractZip').mockRejectedValueOnce(error)
const emitSpy = jest.spyOn(FileEvent, 'emit')
await expect(service.decompress(user, space)).rejects.toBe(error)
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/files/archive')
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')
+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
expect(emitSpy).not.toHaveBeenCalledWith('event', {
user,
@@ -1020,6 +1033,36 @@ describe(FilesManager.name, () => {
rPath: '/data/users/john/files/archive'
})
})
+
+ it('should remove temporary extraction when move fails', async () => {
+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })
+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
+ const error = new Error('move failed')
+ jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)
+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)
+
+ await expect(service.decompress(user, space)).rejects.toBe(error)
+
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')
+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')
+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
+ })
+
+ it('should keep an existing destination when publishing extraction', async () => {
+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })
+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(true)
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
+ jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)
+
+ await expect(service.decompress(user, space)).rejects.toEqual(new FileError(HttpStatus.CONFLICT, 'The destination already exists'))
+
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')
+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ })
})
describe('generateThumbnail', () => {
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 5268dc80..0d4bf73b 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -45,6 +45,7 @@ import {
isPathInside,
isPathIsDir,
makeDir,
+ makeTempDir,
moveFiles,
removeFiles,
touchFile,
@@ -655,9 +656,9 @@ export class FilesManager {
if (!COMPRESSION_EXTENSION.has(extension)) {
throw new FileError(HttpStatus.BAD_REQUEST, `${extension} is not supported`)
}
- // make destination folder
+ // make temporary extraction folder
const dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))
- await makeDir(dstPath)
+ const tmpPath = await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)
let fileLock: FileLock | undefined
try {
// create lock
@@ -669,15 +670,19 @@ export class FilesManager {
}
fileLock = lock
// tasking
- if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath)
+ if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
// do
if (extension === '.zip') {
- await extractZip(space.realPath, dstPath)
+ await extractZip(space.realPath, tmpPath)
} else {
- await extractTar(space.realPath, dstPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)
+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)
}
+ if (await isPathExists(dstPath)) {
+ throw new FileError(HttpStatus.CONFLICT, 'The destination already exists')
+ }
+ await moveFiles(tmpPath, dstPath)
} catch (e) {
- await removeFiles(dstPath).catch((err: Error) => this.logger.error({ tag: this.decompress.name, msg: `unable to remove ${dstPath} : ${err}` }))
+ await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.decompress.name, msg: `unable to remove ${tmpPath} : ${err}` }))
throw e
} finally {
if (fileLock) await this.filesLockManager.removeLock(fileLock.key)
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
index ca8ed7bf..e8c86611 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
@@ -80,6 +80,15 @@ describe(FilesTasksManager.name, () => {
expect(FilesTasksManager.getCacheKey(10)).toBe(`${CACHE_TASK_PREFIX}-10-*`)
})
+ it('should watch a temporary path while keeping the published file name', () => {
+ const startWatchSpy = jest.spyOn(filesTasksManager as any, 'startWatch').mockResolvedValueOnce(undefined)
+ const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any
+
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')
+
+ expect(startWatchSpy).toHaveBeenCalledWith(space, FILE_OPERATION.DECOMPRESS, '/tmp/archive-extract', 'files/personal', 'archive')
+ })
+
it('should create a task and mark it as success when the async method resolves', async () => {
jest.spyOn(crypto, 'randomUUID').mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
filesMethods.doWork.mockResolvedValueOnce({ props: { totalSize: 99 }, result: 'done' })
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.ts
index c832e5e4..bb03875b 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts
@@ -27,8 +27,8 @@ export class FilesTasksManager {
private readonly cache: Cache,
private readonly filesMethods: FilesMethods
) {
- FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string) =>
- this.startWatch(space, taskType, rPath, dirName(space.url))
+ FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string) =>
+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath))
)
}
@@ -144,11 +144,11 @@ export class FilesTasksManager {
}
}
- private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string): Promise<void> {
+ private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string, taskName = fileName(rPath)): Promise<void> {
if (!space.task?.cacheKey || space.task.cacheKey in this.tasksWatcher) return
this.logger.verbose({ tag: this.startWatch.name, msg: `${space.task.cacheKey}` })
this.updateTask(space.task.cacheKey, space.task?.props, {
- name: fileName(rPath),
+ name: taskName,
path: taskPath
}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
switch (taskType) {
diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.ts
index adf131e9..a9cfe7c8 100644
--- a/backend/src/applications/files/utils/files.spec.ts
+++ b/backend/src/applications/files/utils/files.spec.ts
@@ -1,11 +1,11 @@
import { HttpStatus } from '@nestjs/common'
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { Readable } from 'node:stream'
import { FileError } from '../models/file-error'
import { FILE_ERROR_MESSAGES } from './errors'
-import { isPathInside, writeFromStream } from './files'
+import { isPathInside, makeTempDir, writeFromStream } from './files'
describe(isPathInside.name, () => {
const basePath = path.join(path.sep, 'tmp', 'output')
@@ -66,3 +66,25 @@ describe(writeFromStream.name, () => {
await expect(readFile(filePath, 'utf8')).resolves.toBe('abcde')
})
})
+
+describe(makeTempDir.name, () => {
+ let tmpDir: string
+
+ beforeEach(async () => {
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'make-temp-dir-'))
+ })
+
+ afterEach(async () => {
+ await rm(tmpDir, { recursive: true, force: true })
+ })
+
+ it('creates distinct directories with the requested prefix', async () => {
+ const firstPath = await makeTempDir(tmpDir, 'extract-')
+ const secondPath = await makeTempDir(tmpDir, 'extract-')
+
+ expect(firstPath).not.toBe(secondPath)
+ expect(path.basename(firstPath)).toMatch(/^extract-/)
+ await expect(access(firstPath)).resolves.toBeUndefined()
+ await expect(access(secondPath)).resolves.toBeUndefined()
+ })
+})
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index ac3c2c15..8f267acd 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -94,6 +94,11 @@ export function makeDir(rPath: string, recursive?: boolean): Promise<string> {
return fs.mkdir(rPath, { recursive: recursive })
}
+export async function makeTempDir(parentPath: string, prefix: string): Promise<string> {
+ await makeDir(parentPath, true)
+ return fs.mkdtemp(path.join(parentPath, prefix))
+}
+
export function getMimeType(fPath: string, isDir: boolean): string {
if (isDir) {
return 'directory'
diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.ts
index 2911bb8c..12e0a3a3 100644
--- a/backend/src/applications/files/utils/unzip-file.ts
+++ b/backend/src/applications/files/utils/unzip-file.ts
@@ -9,7 +9,7 @@ import { isPathInside, makeDir } from './files'
const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)
export async function extractZip(filePath: string, outputDir: string): Promise<void> {
- const zipFile = await openZipAsync(filePath, { lazyEntries: true })
+ const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })
const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))
const resolvedOutputDir = path.resolve(outputDir)