Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): replace `archiver` with `node-tar` to support file compression and cancellation

johaven7cf27035325c
diff --git a/backend/package.json b/backend/package.json
index d087ff99..1175491b 100755
--- a/backend/package.json
+++ b/backend/package.json
@@ -53,7 +53,6 @@
"@nestjs/websockets": "^11.1.24",
"@socket.io/cluster-adapter": "^0.3.0",
"@socket.io/redis-adapter": "^8.3.0",
- "archiver": "^7.0.1",
"bcryptjs": "^3.0.3",
"class-transformer": "^0.5.1",
"class-validator": "^0.15.1",
@@ -96,7 +95,6 @@
"@nestjs/testing": "^11.1.24",
"@swc/cli": "^0.8.1",
"@swc/core": "^1.15.40",
- "@types/archiver": "^7.0.0",
"@types/bcryptjs": "^2.4.6",
"@types/cookie-parser": "^1.4.10",
"@types/fs-extra": "^11.0.4",
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 33f8280f..6b00aa7a 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -1,11 +1,9 @@
import { HttpService } from '@nestjs/axios'
import { HttpStatus } from '@nestjs/common'
import { Test, TestingModule } from '@nestjs/testing'
-import archiver from 'archiver'
import { lookup } from 'node:dns/promises'
-import fs from 'node:fs'
import path from 'node:path'
-import { PassThrough, Readable } from 'node:stream'
+import { Readable } from 'node:stream'
import { transformAndValidate } from '../../../common/functions'
import * as imageUtils from '../../../common/image'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
@@ -24,6 +22,7 @@ import { SendFile } from '../utils/send-file'
import * as unzipUtils from '../utils/unzip-file'
import * as untarUtils from '../utils/untar-file'
import * as filesUtils from '../utils/files'
+import * as tarUtils from '../utils/tar-file'
import * as taskUtils from '../utils/tasks'
import { FilesLockManager } from './files-lock-manager.service'
import { FilesManager } from './files-manager.service'
@@ -31,11 +30,6 @@ import { FilesQueries } from './files-queries.service'
import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'
import { Mock } from 'vitest'
-vi.mock('archiver', () => ({
- __esModule: true,
- default: vi.fn()
-}))
-
vi.mock('node:dns/promises', () => ({
lookup: vi.fn()
}))
@@ -98,27 +92,6 @@ describe(FilesManager.name, () => {
...overrides
})
- const createArchiveMock = () => {
- const archive = new PassThrough() as PassThrough & {
- directory: Mock
- file: Mock
- finalize: Mock
- abort: Mock
- }
- archive.directory = vi.fn().mockReturnValue(archive)
- archive.file = vi.fn().mockReturnValue(archive)
- archive.finalize = vi.fn().mockImplementation(async () => {
- archive.end()
- })
- archive.abort = vi.fn().mockImplementation(() => {
- archive.end()
- return archive
- })
- vi.mocked(archiver).mockReturnValueOnce(archive as any)
- vi.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any)
- return archive
- }
-
const expectNoWriteOperations = () => {
expect(filesUtils.writeFromStream).not.toHaveBeenCalled()
expect(filesUtils.writeFromStreamAndChecksum).not.toHaveBeenCalled()
@@ -226,6 +199,7 @@ describe(FilesManager.name, () => {
vi.spyOn(taskUtils, 'taskTemporaryPath').mockImplementation((parentPath, cacheKey, name) =>
path.join(parentPath, `.${cacheKey}-${path.basename(name)}`)
)
+ vi.spyOn(tarUtils, 'createTar').mockResolvedValue(undefined)
vi.spyOn(filesUtils, 'getMimeType').mockReturnValue('image-png')
vi.spyOn(spacesPermsUtils, 'canAccessToSpace').mockReturnValue(true)
vi.spyOn(spacesPermsUtils, 'haveSpaceEnvPermissions').mockReturnValue(true)
@@ -1135,16 +1109,15 @@ describe(FilesManager.name, () => {
describe('compress', () => {
it('should archive files and emit events', async () => {
- const archive = createArchiveMock()
- archive.finalize.mockImplementationOnce(async () => {
- archive.end('content')
+ const tarSpy = vi.mocked(tarUtils.createTar).mockImplementationOnce(async (_outputPath, _entries, _gzip, _signal, onProgress) => {
+ onProgress?.(Buffer.byteLength('content'))
})
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')
vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
const dto = {
name: 'archive',
- extension: 'tar.gz',
+ extension: 'tgz',
compressInDirectory: false,
files: [
{ path: '/data/users/john/files/dir', name: 'dir', rootAlias: null },
@@ -1155,25 +1128,17 @@ describe(FilesManager.name, () => {
await service.compress(user, space, dto)
- expect(archiver as unknown as Mock).toHaveBeenCalled()
- expect(archive.directory).toHaveBeenCalled()
- expect(archive.file).toHaveBeenCalled()
- expect(archive.finalize).toHaveBeenCalled()
- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tar.gz')
- expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')
+ expect(tarSpy).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', dto.files, true, undefined, expect.any(Function))
+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tgz')
+ expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tgz')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
- expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {
- highWaterMark: expect.any(Number)
- })
expect(space.task.props.size).toBe(Buffer.byteLength('content'))
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tgz', '/tmp/archive.tgz')
})
it('should allow archive export from trash when compressInDirectory is false', async () => {
- const archive = createArchiveMock()
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive-trash.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid')
- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive-trash.tgz')
+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tgz-compress-uuid')
const emitSpy = vi.spyOn(FileEvent, 'emit')
const space = makeTrashSpace({
url: 'trash/personal/source.txt',
@@ -1182,128 +1147,83 @@ describe(FilesManager.name, () => {
})
const dto = {
name: 'archive-trash',
- extension: 'tar.gz',
+ extension: 'tgz',
compressInDirectory: false,
files: [{ path: '/data/users/john/trash/source.txt', name: 'source.txt', rootAlias: null }]
} as any
await expect(service.compress(user, space, dto)).resolves.toBeUndefined()
expect(filesLockManager.create).not.toHaveBeenCalled()
- expect(archive.file).toHaveBeenCalled()
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid', '/tmp/archive-trash.tar.gz')
- expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tar.gz' })
+ expect(tarUtils.createTar).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tgz-compress-uuid', dto.files, true, undefined, undefined)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tgz-compress-uuid', '/tmp/archive-trash.tgz')
+ expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tgz' })
})
it('should cleanup temporary archive and skip ADD event when publishing archive fails', async () => {
- const archive = createArchiveMock()
const error = new Error('move failed')
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')
+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tgz-compress-uuid')
vi.mocked(filesUtils.moveFiles).mockRejectedValueOnce(error)
const emitSpy = vi.spyOn(FileEvent, 'emit')
const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
const dto = {
name: 'archive',
- extension: 'tar.gz',
+ extension: 'tgz',
compressInDirectory: false,
files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
} as any
await expect(service.compress(user, space, dto)).rejects.toBe(error)
- expect(archive.abort).toHaveBeenCalled()
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.removeFiles).mock.invocationCallOrder[0])
- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tgz-compress-uuid')
+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tgz' })
})
- it('should abort archive pipeline and cleanup temporary file when preparing an entry fails', async () => {
- const archive = createArchiveMock()
- const error = new Error('unable to prepare entry')
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- vi.mocked(filesUtils.isPathIsDir).mockRejectedValueOnce(error)
+ it('should cleanup temporary archive when TAR creation fails', async () => {
+ const error = new Error('archive failed')
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tgz')
+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tgz-compress-uuid')
+ vi.mocked(tarUtils.createTar).mockRejectedValueOnce(error)
const emitSpy = vi.spyOn(FileEvent, 'emit')
const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
const dto = {
name: 'archive',
- extension: 'tar.gz',
+ extension: 'tgz',
compressInDirectory: false,
files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
} as any
await expect(service.compress(user, space, dto)).rejects.toBe(error)
- expect(archive.abort).toHaveBeenCalled()
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(filesUtils.removeFiles).mock.invocationCallOrder[0])
- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tgz-compress-uuid')
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tgz' })
})
- it('should cleanup temporary archive without waiting for finalize after pipeline failure', async () => {
- const archive = createArchiveMock()
- const error = new Error('pipeline failed')
- archive.finalize.mockImplementationOnce(() => {
- archive.emit('error', error)
- return new Promise<void>(() => undefined)
- })
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- vi.mocked(filesUtils.isPathIsDir).mockResolvedValueOnce(false)
- const emitSpy = vi.spyOn(FileEvent, 'emit')
+ it('should preserve task cancellation while cleaning the temporary archive', async () => {
+ const controller = new AbortController()
+ const reason = new Error('Cancelled')
+ vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar')
+ vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar-compress-uuid')
+ vi.mocked(tarUtils.createTar).mockRejectedValueOnce(reason)
const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
const dto = {
name: 'archive',
- extension: 'tar.gz',
+ extension: 'tar',
compressInDirectory: false,
files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
} as any
- await expect(service.compress(user, space, dto)).rejects.toBe(error)
-
- expect(archive.abort).toHaveBeenCalled()
- expect(archive.destroyed).toBe(true)
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
- })
-
- it('should not append entries after archive pipeline cancellation', async () => {
- const archive = createArchiveMock()
- const error = new Error('pipeline failed')
- let resolveEntry!: (isDir: boolean) => void
- let entryStarted!: () => void
- const entryStartedPromise = new Promise<void>((resolve) => {
- entryStarted = resolve
- })
- vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
- vi.mocked(filesUtils.tempFilePath).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
- vi.mocked(filesUtils.isPathIsDir).mockImplementationOnce(
- () =>
- new Promise<boolean>((resolve) => {
- resolveEntry = resolve
- entryStarted()
- })
+ controller.abort(reason)
+ await expect(service.compress(user, space, dto, controller.signal)).rejects.toBe(reason)
+ expect(tarUtils.createTar).toHaveBeenCalledWith(
+ '/data/users/john/tmp/archive.tar-compress-uuid',
+ dto.files,
+ false,
+ controller.signal,
+ undefined
)
- const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
- const dto = {
- name: 'archive',
- extension: 'tar.gz',
- compressInDirectory: false,
- files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
- } as any
-
- const compressPromise = service.compress(user, space, dto)
- await entryStartedPromise
- archive.emit('error', error)
- await new Promise<void>((resolve) => setImmediate(resolve))
-
- expect(filesUtils.removeFiles).not.toHaveBeenCalled()
- resolveEntry(false)
- await expect(compressPromise).rejects.toBe(error)
- expect(archive.file).not.toHaveBeenCalled()
- expect(archive.finalize).not.toHaveBeenCalled()
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar-compress-uuid')
})
})
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 3f86d130..a07ada5c 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -1,7 +1,5 @@
import { HttpService } from '@nestjs/axios'
import { HttpStatus, Injectable, Logger } from '@nestjs/common'
-import archiver, { Archiver } from 'archiver'
-import fs from 'node:fs'
import path from 'node:path'
import { Readable } from 'node:stream'
import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'
@@ -21,8 +19,8 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/pe
import { UserModel } from '../../users/models/user.model'
import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'
import { CACHE_LOCK_FILE_TTL } from '../constants/cache'
-import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
-import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
+import { TAR_GZ_EXTENSION } from '../constants/compress'
+import { COMPRESSION_EXTENSION } from '../constants/files'
import { ALL_DOCUMENT_TYPES, DEFAULT_DOCUMENT_TYPES, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
import { FileDBProps } from '../interfaces/file-db-props.interface'
@@ -35,7 +33,6 @@ import {
copyFileContent,
copyFiles,
createEmptyFile,
- createProgressTransform,
dirName,
dirSize,
fileName,
@@ -63,11 +60,11 @@ import { FilesLockManager } from './files-lock-manager.service'
import { FilesQueries } from './files-queries.service'
import { FileEvent, FileTaskEvent } from '../events/file-events'
import { ACTION } from '../../../common/constants'
-import { pipeline } from 'node:stream/promises'
import { isMultipartFileTooLargeError, uploadTmpFilePath } from '../utils/upload-file'
import { FILE_ERROR_MESSAGES, maxFileSizeExceededError } from '../utils/errors'
import { createTaskTemporaryDir, taskTemporaryPath } from '../utils/tasks'
import { FilesTasksTransfer } from './tasks/files-tasks-transfer.service'
+import { createTar } from '../utils/tar-file'
@Injectable()
export class FilesManager {
@@ -637,13 +634,6 @@ export class FilesManager {
const tmpPath = isTaskContext
? taskTemporaryPath(user.tasksPath, space.task!.cacheKey, dstPath)
: tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)
- // avoid using ZIP here because it can trigger high memory usage.
- const archive: Archiver = archiver(TAR_EXTENSION, {
- gzip: dto.extension === TAR_GZ_EXTENSION,
- gzipOptions: {
- level: 9
- }
- })
// create lock
let fileLock: FileLock | undefined
if (dto.compressInDirectory) {
@@ -661,41 +651,16 @@ export class FilesManager {
FileTaskEvent.emit('startWatch', space, dstPath)
}
// do
- let aborted = false
- let pipePromise: Promise<void> | undefined
- let entriesPromise: Promise<void> | undefined
try {
- const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- if (isTaskContext) {
- const onProgress = this.filesTasksTransfer.createByteProgressHandler(space)
- pipePromise = pipeline(archive, createProgressTransform(onProgress), dstStream, { signal })
- } else {
- pipePromise = pipeline(archive, dstStream, { signal })
- }
- entriesPromise = (async () => {
- for (const f of dto.files) {
- signal?.throwIfAborted()
- const isDir = await isPathIsDir(f.path)
- if (aborted) return
- if (isDir) {
- archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)
- } else {
- archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })
- }
- }
- // The pipeline is the completion signal: finalize() may remain pending after abort().
- if (!aborted) {
- void archive.finalize().catch(() => undefined)
- }
- })()
- await Promise.all([entriesPromise, pipePromise])
- signal?.throwIfAborted()
+ await createTar(
+ tmpPath,
+ dto.files.map((entry) => ({ ...entry, path: entry.path! })),
+ dto.extension === TAR_GZ_EXTENSION,
+ signal,
+ isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined
+ )
await moveFiles(tmpPath, dstPath)
} catch (e) {
- aborted = true
- archive.abort()
- archive.destroy()
- await Promise.allSettled([entriesPromise, pipePromise].filter((promise): promise is Promise<void> => !!promise))
await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.compress.name, msg: `unable to remove ${tmpPath} : ${err}` }))
throw e
} finally {
diff --git a/backend/src/applications/files/utils/tar-file.spec.ts b/backend/src/applications/files/utils/tar-file.spec.ts
new file mode 100644
index 00000000..b7052c26
--- /dev/null
+++ b/backend/src/applications/files/utils/tar-file.spec.ts
@@ -0,0 +1,162 @@
+import { link, mkdir, mkdtemp, rm, symlink, truncate, writeFile } from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import { Writable } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { create, list, type ReadEntry } from 'tar'
+import { createTar } from './tar-file'
+
+const DEFAULT_TEST_FILE_SIZE = 8 * 1024 * 1024
+
+describe(createTar.name, () => {
+ let tmpDir: string
+ let sourceDir: string
+ let archivePath: string
+
+ beforeEach(async () => {
+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'create-tar-'))
+ sourceDir = path.join(tmpDir, 'source')
+ archivePath = path.join(tmpDir, 'archive.tar')
+ await mkdir(sourceDir)
+ })
+
+ afterEach(async () => {
+ vi.restoreAllMocks()
+ await rm(tmpDir, { recursive: true, force: true })
+ })
+
+ it('archives files, directories and symlinks without following link targets', async () => {
+ const directory = path.join(sourceDir, 'docs')
+ const externalFile = path.join(tmpDir, 'external.txt')
+ await mkdir(directory)
+ await writeFile(path.join(directory, 'file.txt'), 'content')
+ await writeFile(externalFile, 'external')
+ await symlink(externalFile, path.join(directory, 'external-link'))
+
+ await createTar(archivePath, [{ path: directory, name: 'docs' }], false)
+
+ const archiveEntries = await readArchiveEntries()
+ expect(archiveEntries.map(({ path }) => path)).toEqual(expect.arrayContaining(['.', 'file.txt', 'external-link']))
+ expect(archiveEntries.find(({ path }) => path === 'external-link')).toMatchObject({
+ type: 'SymbolicLink',
+ linkpath: externalFile
+ })
+ })
+
+ it('preserves selected root names and reports output progress', async () => {
+ const firstFile = path.join(sourceDir, 'first.txt')
+ const secondFile = path.join(sourceDir, 'second.txt')
+ const onProgress = vi.fn()
+ await writeFile(firstFile, 'first')
+ await writeFile(secondFile, 'second')
+
+ await createTar(
+ archivePath,
+ [
+ { path: firstFile, name: 'first.txt' },
+ { path: secondFile, name: 'renamed.txt', rootAlias: 'personal' }
+ ],
+ false,
+ undefined,
+ onProgress
+ )
+
+ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['first.txt', 'renamed.txt'])
+ expect(onProgress).toHaveBeenCalled()
+ })
+
+ it('passes relative source paths to node-tar in strict mode', async () => {
+ const sourceFile = path.join(sourceDir, 'file.txt')
+ await writeFile(sourceFile, 'content')
+
+ await expect(createTar(archivePath, [{ path: sourceFile, name: 'file.txt' }], false)).resolves.toBeUndefined()
+ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['file.txt'])
+ })
+
+ it('filters entries already included in a selected directory', async () => {
+ const directory = path.join(sourceDir, 'docs')
+ const nestedFile = path.join(directory, 'file.txt')
+ await mkdir(directory)
+ await writeFile(nestedFile, 'content')
+
+ await createTar(
+ archivePath,
+ [
+ { path: directory, name: 'docs' },
+ { path: nestedFile, name: 'renamed.txt', rootAlias: 'personal' }
+ ],
+ false
+ )
+
+ expect((await readArchiveEntries()).map(({ path }) => path)).toEqual(['.', 'file.txt'])
+ })
+
+ it('archives hard links as independent files', async () => {
+ const sourceFile = path.join(sourceDir, 'source.txt')
+ const linkedFile = path.join(sourceDir, 'linked.txt')
+ await writeFile(sourceFile, 'content')
+ await link(sourceFile, linkedFile)
+
+ await createTar(
+ archivePath,
+ [
+ { path: sourceFile, name: 'source.txt' },
+ { path: linkedFile, name: 'linked.txt' }
+ ],
+ false
+ )
+
+ expect(await readArchiveEntries()).toEqual([
+ expect.objectContaining({ path: 'source.txt', type: 'File' }),
+ expect.objectContaining({ path: 'linked.txt', type: 'File' })
+ ])
+ })
+
+ it('cancels an active file stream with the original reason', async () => {
+ const largeFile = path.join(sourceDir, 'large.bin')
+ const controller = new AbortController()
+ const reason = new Error('Cancelled')
+ await writeFile(largeFile, '')
+ await truncate(largeFile, DEFAULT_TEST_FILE_SIZE)
+
+ const tarPromise = createTar(archivePath, [{ path: largeFile, name: 'large.bin' }], false, controller.signal, () => {
+ controller.abort(reason)
+ })
+
+ await expect(tarPromise).rejects.toBe(reason)
+ })
+
+ it('detects changes to the node-tar stream API used for cancellation', async () => {
+ const sourceFile = path.join(sourceDir, 'file.txt')
+ await writeFile(sourceFile, 'content')
+ const onWriteEntry = vi.fn((entry) => {
+ expect(entry.absolute).toBe(sourceFile)
+ expect(typeof entry.destroy).toBe('function')
+ })
+ const archive = create({ cwd: sourceDir, onWriteEntry }, ['file.txt'])
+
+ expect(typeof archive.destroy).toBe('function')
+ await pipeline(
+ archive,
+ new Writable({
+ write(_chunk, _encoding, callback) {
+ callback()
+ }
+ })
+ )
+
+ expect(onWriteEntry).toHaveBeenCalledOnce()
+ })
+
+ async function readArchiveEntries(): Promise<Pick<ReadEntry, 'linkpath' | 'path' | 'type'>[]> {
+ const archiveEntries: Pick<ReadEntry, 'linkpath' | 'path' | 'type'>[] = []
+ await list({
+ file: archivePath,
+ onReadEntry: (entry) => {
+ archiveEntries.push({ linkpath: entry.linkpath, path: entry.path, type: entry.type })
+ entry.resume()
+ }
+ })
+ return archiveEntries
+ }
+})
diff --git a/backend/src/applications/files/utils/tar-file.ts b/backend/src/applications/files/utils/tar-file.ts
new file mode 100644
index 00000000..29439fa8
--- /dev/null
+++ b/backend/src/applications/files/utils/tar-file.ts
@@ -0,0 +1,131 @@
+import { createWriteStream } from 'node:fs'
+import { lstat } from 'node:fs/promises'
+import path from 'node:path'
+import { pipeline } from 'node:stream/promises'
+import { create, type WriteEntry } from 'tar'
+import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'
+import { createProgressTransform, fileName } from './files'
+
+export interface TarFileEntry {
+ name: string
+ path: string
+ rootAlias?: string | null
+}
+
+interface TarRoot {
+ archivePath: string
+ realPath: string
+}
+
+function archiveRoots(entries: TarFileEntry[], directoryPaths: Set<string>): TarRoot[] {
+ // Maps each selected filesystem root to the name expected inside the archive.
+ return entries
+ .map((entry) => {
+ const realPath = path.resolve(entry.path)
+ const isDirectory = directoryPaths.has(realPath)
+ const archivePath = isDirectory ? (entries.length === 1 ? '' : fileName(realPath)) : entry.rootAlias ? entry.name : fileName(realPath)
+ return { archivePath, realPath }
+ })
+ .sort((a, b) => b.realPath.length - a.realPath.length)
+}
+
+function archiveEntryPath(realPath: string, roots: TarRoot[]): string {
+ // Converts the absolute path emitted by node-tar to its portable archive path.
+ const root = roots.find(({ realPath: rootPath }) => realPath === rootPath || realPath.startsWith(`${rootPath}${path.sep}`))
+ if (!root) {
+ throw new Error(`Unexpected TAR entry path: ${realPath}`)
+ }
+ const relativePath = path.relative(root.realPath, realPath)
+ return path.posix.join(root.archivePath, relativePath.split(path.sep).join('/')) || './'
+}
+
+function abortReason(signal?: AbortSignal): Error {
+ return signal?.reason instanceof Error ? signal.reason : new Error('Cancelled')
+}
+
+function isPathInside(parentPath: string, candidatePath: string): boolean {
+ const relativePath = path.relative(parentPath, candidatePath)
+ return relativePath !== '' && !path.isAbsolute(relativePath) && relativePath !== '..' && !relativePath.startsWith(`..${path.sep}`)
+}
+
+export async function createTar(
+ outputPath: string,
+ entries: TarFileEntry[],
+ gzip: boolean,
+ signal?: AbortSignal,
+ onProgress?: (bytes: number) => void
+): Promise<void> {
+ // Creates a TAR/TGZ archive while node-tar handles recursion, symlinks and file streaming.
+ signal?.throwIfAborted()
+ if (entries.length === 0) {
+ throw new Error('Cannot create a TAR archive without entries')
+ }
+
+ const resolvedPaths = entries.map(({ path: entryPath }) => path.resolve(entryPath))
+ const stats = await Promise.all(resolvedPaths.map((entryPath) => lstat(entryPath)))
+ signal?.throwIfAborted()
+ const directoryPaths = new Set(resolvedPaths.filter((_entryPath, index) => stats[index].isDirectory()))
+ const selectedEntries = entries
+ .map((entry, index) => ({ entry, realPath: resolvedPaths[index] }))
+ .filter(({ realPath }) => !resolvedPaths.some((parentPath) => directoryPaths.has(parentPath) && isPathInside(parentPath, realPath)))
+ const roots = archiveRoots(
+ selectedEntries.map(({ entry }) => entry),
+ directoryPaths
+ )
+ const cwd = path.parse(selectedEntries[0].realPath).root
+ const sourcePaths = selectedEntries.map(({ realPath: entryPath }) => {
+ const relativePath = path.relative(cwd, entryPath)
+ if (path.isAbsolute(relativePath) || relativePath.startsWith(`..${path.sep}`)) {
+ throw new Error('Cannot create a TAR archive from paths on different filesystem roots')
+ }
+ return relativePath || '.'
+ })
+ let activeEntry: WriteEntry | undefined
+
+ const archive = create(
+ {
+ cwd,
+ filter: (_entryPath, entry) => {
+ // Archive hard links as independent files instead of TAR Link entries.
+ entry.nlink = 1
+ return true
+ },
+ follow: false,
+ gzip: gzip ? { level: 9 } : false,
+ jobs: 1,
+ maxReadSize: DEFAULT_HIGH_WATER_MARK,
+ strict: true,
+ onWriteEntry: (entry) => {
+ activeEntry = entry
+ entry.path = archiveEntryPath(path.resolve(entry.absolute), roots)
+ entry.once('end', () => {
+ if (activeEntry === entry) activeEntry = undefined
+ })
+ }
+ },
+ sourcePaths
+ )
+
+ const onAbort = () => {
+ const reason = abortReason(signal)
+ activeEntry?.destroy(reason)
+ archive.destroy(reason)
+ }
+
+ signal?.addEventListener('abort', onAbort, { once: true })
+ try {
+ const output = createWriteStream(outputPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ if (onProgress) {
+ await pipeline(archive, createProgressTransform(onProgress), output)
+ } else {
+ await pipeline(archive, output)
+ }
+ signal?.throwIfAborted()
+ } catch (error) {
+ if (signal?.aborted) throw abortReason(signal)
+ throw error
+ } finally {
+ signal?.removeEventListener('abort', onAbort)
+ activeEntry?.destroy()
+ }
+}