Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): replace `archiver` with `node-tar` to support file compression and cancellation
johaven7cf27035325c
diff --git a/backend/package.json b/backend/package.jsonindex d087ff99..1175491b 100755--- a/backend/package.json+++ b/backend/package.json@@ -53,7 +53,6 @@"@nestjs/websockets": "^11.1.24","@socket.io/cluster-adapter": "^0.3.0","@socket.io/redis-adapter": "^8.3.0",- "archiver": "^7.0.1","bcryptjs": "^3.0.3","class-transformer": "^0.5.1","class-validator": "^0.15.1",@@ -96,7 +95,6 @@"@nestjs/testing": "^11.1.24","@swc/cli": "^0.8.1","@swc/core": "^1.15.40",- "@types/archiver": "^7.0.0","@types/bcryptjs": "^2.4.6","@types/cookie-parser": "^1.4.10","@types/fs-extra": "^11.0.4",diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 33f8280f..6b00aa7a 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -1,11 +1,9 @@import { HttpService } from '@nestjs/axios'import { HttpStatus } from '@nestjs/common'import { Test, TestingModule } from '@nestjs/testing'-import archiver from 'archiver'import { lookup } from 'node:dns/promises'-import fs from 'node:fs'import path from 'node:path'-import { PassThrough, Readable } from 'node:stream'+import { Readable } from 'node:stream'import { transformAndValidate } from '../../../common/functions'import * as imageUtils from '../../../common/image'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'@@ -24,6 +22,7 @@ import { SendFile } from '../utils/send-file'import * as unzipUtils from '../utils/unzip-file'import * as untarUtils from '../utils/untar-file'import * as filesUtils from '../utils/files'+import * as tarUtils from '../utils/tar-file'import * as taskUtils from '../utils/tasks'import { FilesLockManager } from './files-lock-manager.service'import { FilesManager } from './files-manager.service'@@ -31,11 +30,6 @@ import { FilesQueries } from './files-queries.service'import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'import { Mock } from 'vitest'-vi.mock('archiver', () => ({- __esModule: true,- default: vi.fn()-}))-vi.mock('node:dns/promises', () => ({lookup: vi.fn()}))@@ -98,27 +92,6 @@ describe(FilesManager.name, () => {...overrides})- const createArchiveMock = () => {- const archive = new PassThrough() as PassThrough & {- directory: Mock- file: Mock- finalize: Mock- abort: Mock- }- archive.directory = vi.fn().mockReturnValue(archive)- archive.file = vi.fn().mockReturnValue(archive)- archive.finalize = vi.fn().mockImplementation(async () => {- archive.end()- })- archive.abort = vi.fn().mockImplementation(() => {- archive.end()- return archive- })- vi.mocked(archiver).mockReturnValueOnce(archive as any)- vi.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any)- return archive- }-const expectNoWriteOperations = () => {expect(filesUtils.writeFromStream).not.toHaveBeenCalled()expect(filesUtils.writeFromStreamAndChecksum).not.toHaveBeenCalled()@@ -226,6 +199,7 @@ describe(FilesManager.name, () => {vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>path.join(parentPath, `.${cacheKey}-${path.basename(name)}`))+ vi.spyOn(tarUtils, 'createTar').mockResolvedValue(undefined)vi.spyOn(filesUtils, 'getMimeType').mockReturnValue('image-png')vi.spyOn(spacesPermsUtils, 'canAccessToSpace').mockReturnValue(true)vi.spyOn(spacesPermsUtils, 'haveSpaceEnvPermissions').mockReturnValue(true)@@ -1135,16 +1109,15 @@ describe(FilesManager.name, () => {describe('compress', () => {it('should archive files and emit events', async () => {- const archive = createArchiveMock()- archive.finalize.mockImplementationOnce(async () => {- archive.end('content')+ const tarSpy = vi.mocked(tarUtils.createTar).mockImplementationOnce(async (_outputPath, _entries, _gzip, _signal, onProgress) => {+ onProgress?.(Buffer.byteLength('content'))})- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })const dto = {name: 'archive',- extension: 'tar.gz',+ extension: 'tgz',compressInDirectory: false,files: [{ path: '/data/users/john/files/dir', name: 'dir', rootAlias: null },@@ -1155,25 +1128,17 @@ describe(FilesManager.name, () => {await service.compress(user, space, dto)- expect(archiver as unknown as Mock).toHaveBeenCalled()- expect(archive.directory).toHaveBeenCalled()- expect(archive.file).toHaveBeenCalled()- expect(archive.finalize).toHaveBeenCalled()- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tar.gz')- expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')+ expect(tarSpy).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', dto.files, true, undefined, expect.any(Function))+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tgz')+ expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tgz')expect(filesUtils.tempFilePath).not.toHaveBeenCalled()- expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {- highWaterMark: expect.any(Number)- })expect(space.task.props.size).toBe(Buffer.byteLength('content'))- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', '/tmp/archive.tgz')})it('should allow archive export from trash when compressInDirectory is false', async () => {- const archive = createArchiveMock()- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive-trash.tar.gz')- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid')- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive-trash.tgz')+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tgz-compress-uuid')const emitSpy = vi.spyOn(FileEvent, 'emit')const space = makeTrashSpace({url: 'trash/personal/source.txt',@@ -1182,128 +1147,83 @@ describe(FilesManager.name, () => {})const dto = {name: 'archive-trash',- extension: 'tar.gz',+ extension: 'tgz',compressInDirectory: false,files: [{ path: '/data/users/john/trash/source.txt', name: 'source.txt', rootAlias: null }]} as anyawait expect(service.compress(user, space, dto)).resolves.toBeUndefined()expect(filesLockManager.create).not.toHaveBeenCalled()- expect(archive.file).toHaveBeenCalled()- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid', '/tmp/archive-trash.tar.gz')- expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tar.gz' })+ expect(tarUtils.createTar).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tgz-compress-uuid', dto.files, true, undefined, undefined)+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tgz-compress-uuid', '/tmp/archive-trash.tgz')+ expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tgz' })})it('should cleanup temporary archive and skip ADD event when publishing archive fails', async () => {- const archive = createArchiveMock()const error = new Error('move failed')- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tgz-compress-uuid')vi.mocked(filesUtils.moveFiles).mockRejectedValueOnce(error)const emitSpy = vi.spyOn(FileEvent, 'emit')const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })const dto = {name: 'archive',- extension: 'tar.gz',+ extension: 'tgz',compressInDirectory: false,files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]} as anyawait expect(service.compress(user, space, dto)).rejects.toBe(error)- expect(archive.abort).toHaveBeenCalled()- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')- expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.removeFiles).mock.invocationCallOrder[0])- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tgz-compress-uuid')+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tgz' })})- it('should abort archive pipeline and cleanup temporary file when preparing an entry fails', async () => {- const archive = createArchiveMock()- const error = new Error('unable to prepare entry')- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')- vi.mocked(filesUtils.isPathIsDir).mockRejectedValueOnce(error)+ it('should cleanup temporary archive when TAR creation fails', async () => {+ const error = new Error('archive failed')+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tgz-compress-uuid')+ vi.mocked(tarUtils.createTar).mockRejectedValueOnce(error)const emitSpy = vi.spyOn(FileEvent, 'emit')const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })const dto = {name: 'archive',- extension: 'tar.gz',+ extension: 'tgz',compressInDirectory: false,files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]} as anyawait expect(service.compress(user, space, dto)).rejects.toBe(error)- expect(archive.abort).toHaveBeenCalled()- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')- expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.removeFiles).mock.invocationCallOrder[0])- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tgz-compress-uuid')+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tgz' })})- it('should cleanup temporary archive without waiting for finalize after pipeline failure', async () => {- const archive = createArchiveMock()- const error = new Error('pipeline failed')- archive.finalize.mockImplementationOnce(() => {- archive.emit('error', error)- return new Promise<void>(() => undefined)- })- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)- const emitSpy = vi.spyOn(FileEvent, 'emit')+ it('should preserve task cancellation while cleaning the temporary archive', async () => {+ const controller = new AbortController()+ const reason = new Error('Cancelled')+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar')+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar-compress-uuid')+ vi.mocked(tarUtils.createTar).mockRejectedValueOnce(reason)const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })const dto = {name: 'archive',- extension: 'tar.gz',+ extension: 'tar',compressInDirectory: false,files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]} as any- await expect(service.compress(user, space, dto)).rejects.toBe(error)-- expect(archive.abort).toHaveBeenCalled()- expect(archive.destroyed).toBe(true)- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })- })-- it('should not append entries after archive pipeline cancellation', async () => {- const archive = createArchiveMock()- const error = new Error('pipeline failed')- let resolveEntry!: (isDir: boolean) => void- let entryStarted!: () => void- const entryStartedPromise = new Promise<void>((resolve) => {- entryStarted = resolve- })- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')- vi.mocked(filesUtils.isPathIsDir).mockImplementationOnce(- () =>- new Promise<boolean>((resolve) => {- resolveEntry = resolve- entryStarted()- })+ controller.abort(reason)+ await expect(service.compress(user, space, dto, controller.signal)).rejects.toBe(reason)+ expect(tarUtils.createTar).toHaveBeenCalledWith(+ '/data/users/john/tmp/archive.tar-compress-uuid',+ dto.files,+ false,+ controller.signal,+ undefined)- const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })- const dto = {- name: 'archive',- extension: 'tar.gz',- compressInDirectory: false,- files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]- } as any-- const compressPromise = service.compress(user, space, dto)- await entryStartedPromise- archive.emit('error', error)- await new Promise<void>((resolve) => setImmediate(resolve))-- expect(filesUtils.removeFiles).not.toHaveBeenCalled()- resolveEntry(false)- await expect(compressPromise).rejects.toBe(error)- expect(archive.file).not.toHaveBeenCalled()- expect(archive.finalize).not.toHaveBeenCalled()- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar-compress-uuid')})})diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 3f86d130..a07ada5c 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -1,7 +1,5 @@import { HttpService } from '@nestjs/axios'import { HttpStatus, Injectable, Logger } from '@nestjs/common'-import archiver, { Archiver } from 'archiver'-import fs from 'node:fs'import path from 'node:path'import { Readable } from 'node:stream'import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'@@ -21,8 +19,8 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/peimport { UserModel } from '../../users/models/user.model'import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'import { CACHE_LOCK_FILE_TTL } from '../constants/cache'-import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'-import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'+import { TAR_GZ_EXTENSION } from '../constants/compress'+import { COMPRESSION_EXTENSION } from '../constants/files'import { ALL_DOCUMENT_TYPES, DEFAULT_DOCUMENT_TYPES, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'import { FileDBProps } from '../interfaces/file-db-props.interface'@@ -35,7 +33,6 @@ import {copyFileContent,copyFiles,createEmptyFile,- createProgressTransform,dirName,dirSize,fileName,@@ -63,11 +60,11 @@ import { FilesLockManager } from './files-lock-manager.service'import { FilesQueries } from './files-queries.service'import { FileEvent, FileTaskEvent } from '../events/file-events'import { ACTION } from '../../../common/constants'-import { pipeline } from 'node:stream/promises'import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'import { FILE_ERROR_MESSAGES, maxFileSizeExceededError } from '../utils/errors'import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'+import { createTar } from '../utils/tar-file'@Injectable()export class FilesManager {@@ -637,13 +634,6 @@ export class FilesManager {const tmpPath = isTaskContext? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath): tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)- // avoid using ZIP here because it can trigger high memory usage.- const archive: Archiver = archiver(TAR_EXTENSION, {- gzip: dto.extension === TAR_GZ_EXTENSION,- gzipOptions: {- level: 9- }- })// create locklet fileLock: FileLock | undefinedif (dto.compressInDirectory) {@@ -661,41 +651,16 @@ export class FilesManager {FileTaskEvent.emit('startWatch', space, dstPath)}// do- let aborted = false- let pipePromise: Promise<void> | undefined- let entriesPromise: Promise<void> | undefinedtry {- const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- if (isTaskContext) {- const onProgress = this.filesTasksTransfer.createByteProgressHandler(space)- pipePromise = pipeline(archive, createProgressTransform(onProgress), dstStream, { signal })- } else {- pipePromise = pipeline(archive, dstStream, { signal })- }- entriesPromise = (async () => {- for (const f of dto.files) {- signal?.throwIfAborted()- const isDir = await isPathIsDir(f.path)- if (aborted) return- if (isDir) {- archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)- } else {- archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })- }- }- // The pipeline is the completion signal: finalize() may remain pending after abort().- if (!aborted) {- void archive.finalize().catch(() => undefined)- }- })()- await Promise.all([entriesPromise, pipePromise])- signal?.throwIfAborted()+ await createTar(+ tmpPath,+ dto.files.map((entry) => ({ ...entry, path: entry.path! })),+ dto.extension === TAR_GZ_EXTENSION,+ signal,+ isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined+ )await moveFiles(tmpPath, dstPath)} catch (e) {- aborted = true- archive.abort()- archive.destroy()- await Promise.allSettled([entriesPromise, pipePromise].filter((promise): promise is Promise<void> => !!promise))await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.compress.name, msg: `unable to remove ${tmpPath} : ${err}` }))throw e} finally {diff --git a/backend/src/applications/files/utils/tar-file.spec.ts b/backend/src/applications/files/utils/tar-file.spec.tsnew file mode 100644index 00000000..b7052c26--- /dev/null+++ b/backend/src/applications/files/utils/tar-file.spec.ts@@ -0,0 +1,162 @@+import { link, mkdir, mkdtemp, rm, symlink, truncate, writeFile } from 'node:fs/promises'+import os from 'node:os'+import path from 'node:path'+import { Writable } from 'node:stream'+import { pipeline } from 'node:stream/promises'+import { create, list, type ReadEntry } from 'tar'+import { createTar } from './tar-file'++const DEFAULT_TEST_FILE_SIZE = 8 * 1024 * 1024++describe(createTar.name, () => {+ let tmpDir: string+ let sourceDir: string+ let archivePath: string++ beforeEach(async () => {+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'create-tar-'))+ sourceDir = path.join(tmpDir, 'source')+ archivePath = path.join(tmpDir, 'archive.tar')+ await mkdir(sourceDir)+ })++ afterEach(async () => {+ vi.restoreAllMocks()+ await rm(tmpDir, { recursive: true, force: true })+ })++ it('archives files, directories and symlinks without following link targets', async () => {+ const directory = path.join(sourceDir, 'docs')+ const externalFile = path.join(tmpDir, 'external.txt')+ await mkdir(directory)+ await writeFile(path.join(directory, 'file.txt'), 'content')+ await writeFile(externalFile, 'external')+ await symlink(externalFile, path.join(directory, 'external-link'))++ await createTar(archivePath, [{ path: directory, name: 'docs' }], false)++ const archiveEntries = await readArchiveEntries()+ expect(archiveEntries.map(({ path }) => path)).toEqual(expect.arrayContaining(['.', 'file.txt', 'external-link']))+ expect(archiveEntries.find(({ path }) => path === 'external-link')).toMatchObject({+ type: 'SymbolicLink',+ linkpath: externalFile+ })+ })++ it('preserves selected root names and reports output progress', async () => {+ const firstFile = path.join(sourceDir, 'first.txt')+ const secondFile = path.join(sourceDir, 'second.txt')+ const onProgress = vi.fn()+ await writeFile(firstFile, 'first')+ await writeFile(secondFile, 'second')++ await createTar(+ archivePath,+ [+ { path: firstFile, name: 'first.txt' },+ { path: secondFile, name: 'renamed.txt', rootAlias: 'personal' }+ ],+ false,+ undefined,+ onProgress+ )++ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['first.txt', 'renamed.txt'])+ expect(onProgress).toHaveBeenCalled()+ })++ it('passes relative source paths to node-tar in strict mode', async () => {+ const sourceFile = path.join(sourceDir, 'file.txt')+ await writeFile(sourceFile, 'content')++ await expect(createTar(archivePath, [{ path: sourceFile, name: 'file.txt' }], false)).resolves.toBeUndefined()+ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['file.txt'])+ })++ it('filters entries already included in a selected directory', async () => {+ const directory = path.join(sourceDir, 'docs')+ const nestedFile = path.join(directory, 'file.txt')+ await mkdir(directory)+ await writeFile(nestedFile, 'content')++ await createTar(+ archivePath,+ [+ { path: directory, name: 'docs' },+ { path: nestedFile, name: 'renamed.txt', rootAlias: 'personal' }+ ],+ false+ )++ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['.', 'file.txt'])+ })++ it('archives hard links as independent files', async () => {+ const sourceFile = path.join(sourceDir, 'source.txt')+ const linkedFile = path.join(sourceDir, 'linked.txt')+ await writeFile(sourceFile, 'content')+ await link(sourceFile, linkedFile)++ await createTar(+ archivePath,+ [+ { path: sourceFile, name: 'source.txt' },+ { path: linkedFile, name: 'linked.txt' }+ ],+ false+ )++ expect(await readArchiveEntries()).toEqual([+ expect.objectContaining({ path: 'source.txt', type: 'File' }),+ expect.objectContaining({ path: 'linked.txt', type: 'File' })+ ])+ })++ it('cancels an active file stream with the original reason', async () => {+ const largeFile = path.join(sourceDir, 'large.bin')+ const controller = new AbortController()+ const reason = new Error('Cancelled')+ await writeFile(largeFile, '')+ await truncate(largeFile, DEFAULT_TEST_FILE_SIZE)++ const tarPromise = createTar(archivePath, [{ path: largeFile, name: 'large.bin' }], false, controller.signal, () => {+ controller.abort(reason)+ })++ await expect(tarPromise).rejects.toBe(reason)+ })++ it('detects changes to the node-tar stream API used for cancellation', async () => {+ const sourceFile = path.join(sourceDir, 'file.txt')+ await writeFile(sourceFile, 'content')+ const onWriteEntry = vi.fn((entry) => {+ expect(entry.absolute).toBe(sourceFile)+ expect(typeof entry.destroy).toBe('function')+ })+ const archive = create({ cwd: sourceDir, onWriteEntry }, ['file.txt'])++ expect(typeof archive.destroy).toBe('function')+ await pipeline(+ archive,+ new Writable({+ write(_chunk, _encoding, callback) {+ callback()+ }+ })+ )++ expect(onWriteEntry).toHaveBeenCalledOnce()+ })++ async function readArchiveEntries(): Promise<Pick<ReadEntry, 'linkpath' | 'path' | 'type'>[]> {+ const archiveEntries: Pick<ReadEntry, 'linkpath' | 'path' | 'type'>[] = []+ await list({+ file: archivePath,+ onReadEntry: (entry) => {+ archiveEntries.push({ linkpath: entry.linkpath, path: entry.path, type: entry.type })+ entry.resume()+ }+ })+ return archiveEntries+ }+})diff --git a/backend/src/applications/files/utils/tar-file.ts b/backend/src/applications/files/utils/tar-file.tsnew file mode 100644index 00000000..29439fa8--- /dev/null+++ b/backend/src/applications/files/utils/tar-file.ts@@ -0,0 +1,131 @@+import { createWriteStream } from 'node:fs'+import { lstat } from 'node:fs/promises'+import path from 'node:path'+import { pipeline } from 'node:stream/promises'+import { create, type WriteEntry } from 'tar'+import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'+import { createProgressTransform, fileName } from './files'++export interface TarFileEntry {+ name: string+ path: string+ rootAlias?: string | null+}++interface TarRoot {+ archivePath: string+ realPath: string+}++function archiveRoots(entries: TarFileEntry[], directoryPaths: Set<string>): TarRoot[] {+ // Maps each selected filesystem root to the name expected inside the archive.+ return entries+ .map((entry) => {+ const realPath = path.resolve(entry.path)+ const isDirectory = directoryPaths.has(realPath)+ const archivePath = isDirectory ? (entries.length === 1 ? '' : fileName(realPath)) : entry.rootAlias ? entry.name : fileName(realPath)+ return { archivePath, realPath }+ })+ .sort((a, b) => b.realPath.length - a.realPath.length)+}++function archiveEntryPath(realPath: string, roots: TarRoot[]): string {+ // Converts the absolute path emitted by node-tar to its portable archive path.+ const root = roots.find(({ realPath: rootPath }) => realPath === rootPath || realPath.startsWith(`${rootPath}${path.sep}`))+ if (!root) {+ throw new Error(`Unexpected TAR entry path: ${realPath}`)+ }+ const relativePath = path.relative(root.realPath, realPath)+ return path.posix.join(root.archivePath, relativePath.split(path.sep).join('/')) || './'+}++function abortReason(signal?: AbortSignal): Error {+ return signal?.reason instanceof Error ? signal.reason : new Error('Cancelled')+}++function isPathInside(parentPath: string, candidatePath: string): boolean {+ const relativePath = path.relative(parentPath, candidatePath)+ return relativePath !== '' && !path.isAbsolute(relativePath) && relativePath !== '..' && !relativePath.startsWith(`..${path.sep}`)+}++export async function createTar(+ outputPath: string,+ entries: TarFileEntry[],+ gzip: boolean,+ signal?: AbortSignal,+ onProgress?: (bytes: number) => void+): Promise<void> {+ // Creates a TAR/TGZ archive while node-tar handles recursion, symlinks and file streaming.+ signal?.throwIfAborted()+ if (entries.length === 0) {+ throw new Error('Cannot create a TAR archive without entries')+ }++ const resolvedPaths = entries.map(({ path: entryPath }) => path.resolve(entryPath))+ const stats = await Promise.all(resolvedPaths.map((entryPath) => lstat(entryPath)))+ signal?.throwIfAborted()+ const directoryPaths = new Set(resolvedPaths.filter((_entryPath, index) => stats[index].isDirectory()))+ const selectedEntries = entries+ .map((entry, index) => ({ entry, realPath: resolvedPaths[index] }))+ .filter(({ realPath }) => !resolvedPaths.some((parentPath) => directoryPaths.has(parentPath) && isPathInside(parentPath, realPath)))+ const roots = archiveRoots(+ selectedEntries.map(({ entry }) => entry),+ directoryPaths+ )+ const cwd = path.parse(selectedEntries[0].realPath).root+ const sourcePaths = selectedEntries.map(({ realPath: entryPath }) => {+ const relativePath = path.relative(cwd, entryPath)+ if (path.isAbsolute(relativePath) || relativePath.startsWith(`..${path.sep}`)) {+ throw new Error('Cannot create a TAR archive from paths on different filesystem roots')+ }+ return relativePath || '.'+ })+ let activeEntry: WriteEntry | undefined++ const archive = create(+ {+ cwd,+ filter: (_entryPath, entry) => {+ // Archive hard links as independent files instead of TAR Link entries.+ entry.nlink = 1+ return true+ },+ follow: false,+ gzip: gzip ? { level: 9 } : false,+ jobs: 1,+ maxReadSize: DEFAULT_HIGH_WATER_MARK,+ strict: true,+ onWriteEntry: (entry) => {+ activeEntry = entry+ entry.path = archiveEntryPath(path.resolve(entry.absolute), roots)+ entry.once('end', () => {+ if (activeEntry === entry) activeEntry = undefined+ })+ }+ },+ sourcePaths+ )++ const onAbort = () => {+ const reason = abortReason(signal)+ activeEntry?.destroy(reason)+ archive.destroy(reason)+ }++ signal?.addEventListener('abort', onAbort, { once: true })+ try {+ const output = createWriteStream(outputPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })+ if (onProgress) {+ await pipeline(archive, createProgressTransform(onProgress), output)+ } else {+ await pipeline(archive, output)+ }+ signal?.throwIfAborted()+ } catch (error) {+ if (signal?.aborted) throw abortReason(signal)+ throw error+ } finally {+ signal?.removeEventListener('abort', onAbort)+ activeEntry?.destroy()+ }+}