Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): compress method

johavenc3625e5e0d29
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 65cd420c..aaaa988b 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -4,7 +4,7 @@ import { Test, TestingModule } from '@nestjs/testing'
import archiver from 'archiver'
import fs from 'node:fs'
import path from 'node:path'
-import { Readable } from 'node:stream'
+import { PassThrough, Readable } from 'node:stream'
import * as tar from 'tar'
import * as imageUtils from '../../../common/image'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
@@ -382,15 +382,18 @@ describe(FilesManager.name, () => {
})
it('compress should archive files and emit events', async () => {
- const archive = {
- on: jest.fn().mockReturnThis(),
- pipe: jest.fn().mockReturnThis(),
- directory: jest.fn().mockReturnThis(),
- file: jest.fn().mockReturnThis(),
- finalize: jest.fn().mockResolvedValue(undefined)
+ const archive = new PassThrough() as PassThrough & {
+ directory: jest.Mock
+ file: jest.Mock
+ finalize: jest.Mock
}
- ;(archiver as unknown as jest.Mock).mockReturnValueOnce(archive)
- jest.spyOn(fs, 'createWriteStream').mockReturnValue({} as any)
+ archive.directory = jest.fn().mockReturnValue(archive)
+ archive.file = jest.fn().mockReturnValue(archive)
+ archive.finalize = jest.fn().mockImplementation(async () => {
+ archive.end()
+ })
+ ;(archiver as unknown as jest.Mock).mockReturnValueOnce(archive as any)
+ jest.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any)
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
;(filesUtils.isPathIsDir as jest.Mock).mockImplementation(async (p: string) => p.endsWith('/dir'))
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index ce8f4c58..7f7677b5 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -1,6 +1,6 @@
import { HttpService } from '@nestjs/axios'
import { HttpStatus, Injectable, Logger } from '@nestjs/common'
-import archiver, { Archiver, ArchiverError } from 'archiver'
+import archiver, { Archiver } from 'archiver'
import { AxiosResponse } from 'axios'
import fs from 'node:fs'
import path from 'node:path'
@@ -23,7 +23,7 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/pe
import { UserModel } from '../../users/models/user.model'
import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'
import { CACHE_LOCK_FILE_TTL } from '../constants/cache'
-import { TAR_GZ_EXTENSION } from '../constants/compress'
+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
import { FILE_OPERATION } from '../constants/operations'
import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
@@ -61,6 +61,7 @@ import { FilesLockManager } from './files-lock-manager.service'
import { FilesQueries } from './files-queries.service'
import { FileEvent, FileTaskEvent } from '../events/file-events'
import { ACTION } from '../../../common/constants'
+import { pipeline } from 'node:stream/promises'
@Injectable()
export class FilesManager {
@@ -562,10 +563,10 @@ export class FilesManager {
// This method is currently used only by files-methods.service, which handles input sanitization.
// If it is used in other services in the future, make sure to refactor accordingly to sanitize inputs properly.
const srcPath = dirName(space.realPath)
- // todo: a guest link tasksPath should be in specific directory (guest link has no home)
const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`
const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))
- const archive: Archiver = archiver('tar', {
+ // avoid using ZIP here because it can trigger high memory usage.
+ const archive: Archiver = archiver(TAR_EXTENSION, {
gzip: dto.extension === TAR_GZ_EXTENSION,
gzipOptions: {
level: 9
@@ -588,21 +589,19 @@ export class FilesManager {
}
// do
try {
- archive.on('error', (error: ArchiverError) => {
- throw error
- })
const dstStream = fs.createWriteStream(dstPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- archive.pipe(dstStream)
+ const pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream
+
for (const f of dto.files) {
if (await isPathIsDir(f.path)) {
archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)
} else {
- archive.file(f.path, {
- name: f.rootAlias ? f.name : fileName(f.path)
- })
+ archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })
}
}
- await archive.finalize()
+
+ const finalizePromise = archive.finalize()
+ await Promise.all([finalizePromise, pipePromise])
} finally {
if (fileLock) {
await this.filesLockManager.removeLock(fileLock.key)