Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): stage archive extraction in user temp directory
johaven06f142564a56
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.tsindex 42bb61dc..b90bab84 100644--- a/backend/src/applications/files/interfaces/file-event.interface.ts+++ b/backend/src/applications/files/interfaces/file-event.interface.ts@@ -4,7 +4,7 @@ import type { ACTION } from '../../../common/constants'import type { UserModel } from '../../users/models/user.model'export interface FileTaskEventEmit {- startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string]+ startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string]}export interface FileEventType {diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex e0cf7011..c5c48e22 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -164,6 +164,7 @@ describe(FilesManager.name, () => {jest.spyOn(filesUtils, 'isPathExists').mockResolvedValue(true)jest.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)jest.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)+ jest.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')jest.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)jest.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')jest.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)@@ -977,41 +978,53 @@ describe(FilesManager.name, () => {describe('decompress', () => {it('should extract zip and release lock', async () => {const space = makeSpace({ realPath: '/data/users/john/files/archive.zip', task: { cacheKey: 'task-d', props: {} } })- ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false);(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')const unzipSpy = jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)const taskEmitSpy = jest.spyOn(FileTaskEvent, 'emit')await service.decompress(user, space)- expect(filesUtils.makeDir).toHaveBeenCalledWith('/data/users/john/files/archive')- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/files/archive')- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DECOMPRESS, '/data/users/john/files/archive')+ expect(filesUtils.makeTempDir).toHaveBeenCalledWith('/data/users/john/tmp', 'archive-extract-')+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123')+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')+ expect(taskEmitSpy).toHaveBeenCalledWith(+ 'startWatch',+ space,+ FILE_OPERATION.DECOMPRESS,+ '/data/users/john/files/archive',+ '/data/users/john/tmp/archive-extract-123'+ )expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')})it('should extract tar formats via extractTar', async () => {const space = makeSpace({ realPath: '/data/users/john/files/archive.tar.gz' })- ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false);(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')const untarSpy = jest.spyOn(untarUtils, 'extractTar').mockResolvedValueOnce(undefined)await service.decompress(user, space)- expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/files/archive', true)+ expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true)+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')})it('should remove partial extraction and skip add event on failure', async () => {const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' });(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true);(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')const error = new Error('extraction failed')jest.spyOn(unzipUtils, 'extractZip').mockRejectedValueOnce(error)const emitSpy = jest.spyOn(FileEvent, 'emit')await expect(service.decompress(user, space)).rejects.toBe(error)- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/files/archive')+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')expect(emitSpy).not.toHaveBeenCalledWith('event', {user,@@ -1020,6 +1033,36 @@ describe(FilesManager.name, () => {rPath: '/data/users/john/files/archive'})})++ it('should remove temporary extraction when move fails', async () => {+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')+ const error = new Error('move failed')+ jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)++ await expect(service.decompress(user, space)).rejects.toBe(error)++ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')+ })++ it('should keep an existing destination when publishing extraction', async () => {+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(true)+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')+ jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)++ await expect(service.decompress(user, space)).rejects.toEqual(new FileError(HttpStatus.CONFLICT, 'The destination already exists'))++ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123')+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith('/data/users/john/files/archive')+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()+ })})describe('generateThumbnail', () => {diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 5268dc80..0d4bf73b 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -45,6 +45,7 @@ import {isPathInside,isPathIsDir,makeDir,+ makeTempDir,moveFiles,removeFiles,touchFile,@@ -655,9 +656,9 @@ export class FilesManager {if (!COMPRESSION_EXTENSION.has(extension)) {throw new FileError(HttpStatus.BAD_REQUEST, `${extension} is not supported`)}- // make destination folder+ // make temporary extraction folderconst dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))- await makeDir(dstPath)+ const tmpPath = await makeTempDir(user.tmpPath, `${fileName(dstPath)}-extract-`)let fileLock: FileLock | undefinedtry {// create lock@@ -669,15 +670,19 @@ export class FilesManager {}fileLock = lock// tasking- if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath)+ if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)// doif (extension === '.zip') {- await extractZip(space.realPath, dstPath)+ await extractZip(space.realPath, tmpPath)} else {- await extractTar(space.realPath, dstPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)}+ if (await isPathExists(dstPath)) {+ throw new FileError(HttpStatus.CONFLICT, 'The destination already exists')+ }+ await moveFiles(tmpPath, dstPath)} catch (e) {- await removeFiles(dstPath).catch((err: Error) => this.logger.error({ tag: this.decompress.name, msg: `unable to remove ${dstPath} : ${err}` }))+ await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.decompress.name, msg: `unable to remove ${tmpPath} : ${err}` }))throw e} finally {if (fileLock) await this.filesLockManager.removeLock(fileLock.key)diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.tsindex ca8ed7bf..e8c86611 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts@@ -80,6 +80,15 @@ describe(FilesTasksManager.name, () => {expect(FilesTasksManager.getCacheKey(10)).toBe(`${CACHE_TASK_PREFIX}-10-*`)})+ it('should watch a temporary path while keeping the published file name', () => {+ const startWatchSpy = jest.spyOn(filesTasksManager as any, 'startWatch').mockResolvedValueOnce(undefined)+ const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any++ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')++ expect(startWatchSpy).toHaveBeenCalledWith(space, FILE_OPERATION.DECOMPRESS, '/tmp/archive-extract', 'files/personal', 'archive')+ })+it('should create a task and mark it as success when the async method resolves', async () => {jest.spyOn(crypto, 'randomUUID').mockReturnValueOnce('11111111-1111-4111-8111-111111111111')filesMethods.doWork.mockResolvedValueOnce({ props: { totalSize: 99 }, result: 'done' })diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.tsindex c832e5e4..bb03875b 100644--- a/backend/src/applications/files/services/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts@@ -27,8 +27,8 @@ export class FilesTasksManager {private readonly cache: Cache,private readonly filesMethods: FilesMethods) {- FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string) =>- this.startWatch(space, taskType, rPath, dirName(space.url))+ FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string) =>+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath)))}@@ -144,11 +144,11 @@ export class FilesTasksManager {}}- private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string): Promise<void> {+ private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string, taskName = fileName(rPath)): Promise<void> {if (!space.task?.cacheKey || space.task.cacheKey in this.tasksWatcher) returnthis.logger.verbose({ tag: this.startWatch.name, msg: `${space.task.cacheKey}` })this.updateTask(space.task.cacheKey, space.task?.props, {- name: fileName(rPath),+ name: taskName,path: taskPath}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))switch (taskType) {diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.tsindex adf131e9..a9cfe7c8 100644--- a/backend/src/applications/files/utils/files.spec.ts+++ b/backend/src/applications/files/utils/files.spec.ts@@ -1,11 +1,11 @@import { HttpStatus } from '@nestjs/common'-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'+import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { Readable } from 'node:stream'import { FileError } from '../models/file-error'import { FILE_ERROR_MESSAGES } from './errors'-import { isPathInside, writeFromStream } from './files'+import { isPathInside, makeTempDir, writeFromStream } from './files'describe(isPathInside.name, () => {const basePath = path.join(path.sep, 'tmp', 'output')@@ -66,3 +66,25 @@ describe(writeFromStream.name, () => {await expect(readFile(filePath, 'utf8')).resolves.toBe('abcde')})})++describe(makeTempDir.name, () => {+ let tmpDir: string++ beforeEach(async () => {+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'make-temp-dir-'))+ })++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true })+ })++ it('creates distinct directories with the requested prefix', async () => {+ const firstPath = await makeTempDir(tmpDir, 'extract-')+ const secondPath = await makeTempDir(tmpDir, 'extract-')++ expect(firstPath).not.toBe(secondPath)+ expect(path.basename(firstPath)).toMatch(/^extract-/)+ await expect(access(firstPath)).resolves.toBeUndefined()+ await expect(access(secondPath)).resolves.toBeUndefined()+ })+})diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex ac3c2c15..8f267acd 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -94,6 +94,11 @@ export function makeDir(rPath: string, recursive?: boolean): Promise<string> {return fs.mkdir(rPath, { recursive: recursive })}+export async function makeTempDir(parentPath: string, prefix: string): Promise<string> {+ await makeDir(parentPath, true)+ return fs.mkdtemp(path.join(parentPath, prefix))+}+export function getMimeType(fPath: string, isDir: boolean): string {if (isDir) {return 'directory'diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.tsindex 2911bb8c..12e0a3a3 100644--- a/backend/src/applications/files/utils/unzip-file.ts+++ b/backend/src/applications/files/utils/unzip-file.ts@@ -9,7 +9,7 @@ import { isPathInside, makeDir } from './files'const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)export async function extractZip(filePath: string, outputDir: string): Promise<void> {- const zipFile = await openZipAsync(filePath, { lazyEntries: true })+ const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))const resolvedOutputDir = path.resolve(outputDir)