Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): compress method
johavenc3625e5e0d29
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 65cd420c..aaaa988b 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -4,7 +4,7 @@ import { Test, TestingModule } from '@nestjs/testing'import archiver from 'archiver'import fs from 'node:fs'import path from 'node:path'-import { Readable } from 'node:stream'+import { PassThrough, Readable } from 'node:stream'import * as tar from 'tar'import * as imageUtils from '../../../common/image'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'@@ -382,15 +382,18 @@ describe(FilesManager.name, () => {})it('compress should archive files and emit events', async () => {- const archive = {- on: jest.fn().mockReturnThis(),- pipe: jest.fn().mockReturnThis(),- directory: jest.fn().mockReturnThis(),- file: jest.fn().mockReturnThis(),- finalize: jest.fn().mockResolvedValue(undefined)+ const archive = new PassThrough() as PassThrough & {+ directory: jest.Mock+ file: jest.Mock+ finalize: jest.Mock}- ;(archiver as unknown as jest.Mock).mockReturnValueOnce(archive)- jest.spyOn(fs, 'createWriteStream').mockReturnValue({} as any)+ archive.directory = jest.fn().mockReturnValue(archive)+ archive.file = jest.fn().mockReturnValue(archive)+ archive.finalize = jest.fn().mockImplementation(async () => {+ archive.end()+ })+ ;(archiver as unknown as jest.Mock).mockReturnValueOnce(archive as any)+ jest.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any);(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz');(filesUtils.isPathIsDir as jest.Mock).mockImplementation(async (p: string) => p.endsWith('/dir'))const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex ce8f4c58..7f7677b5 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -1,6 +1,6 @@import { HttpService } from '@nestjs/axios'import { HttpStatus, Injectable, Logger } from '@nestjs/common'-import archiver, { Archiver, ArchiverError } from 'archiver'+import archiver, { Archiver } from 'archiver'import { AxiosResponse } from 'axios'import fs from 'node:fs'import path from 'node:path'@@ -23,7 +23,7 @@ import { canAccessToSpace, haveSpaceEnvPermissions } from '../../spaces/utils/peimport { UserModel } from '../../users/models/user.model'import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'import { CACHE_LOCK_FILE_TTL } from '../constants/cache'-import { TAR_GZ_EXTENSION } from '../constants/compress'+import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'import { FILE_OPERATION } from '../constants/operations'import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'@@ -61,6 +61,7 @@ import { FilesLockManager } from './files-lock-manager.service'import { FilesQueries } from './files-queries.service'import { FileEvent, FileTaskEvent } from '../events/file-events'import { ACTION } from '../../../common/constants'+import { pipeline } from 'node:stream/promises'@Injectable()export class FilesManager {@@ -562,10 +563,10 @@ export class FilesManager {// This method is currently used only by files-methods.service, which handles input sanitization.// If it is used in other services in the future, make sure to refactor accordingly to sanitize inputs properly.const srcPath = dirName(space.realPath)- // todo: a guest link tasksPath should be in specific directory (guest link has no home)const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))- const archive: Archiver = archiver('tar', {+ // avoid using ZIP here because it can trigger high memory usage.+ const archive: Archiver = archiver(TAR_EXTENSION, {gzip: dto.extension === TAR_GZ_EXTENSION,gzipOptions: {level: 9@@ -588,21 +589,19 @@ export class FilesManager {}// dotry {- archive.on('error', (error: ArchiverError) => {- throw error- })const dstStream = fs.createWriteStream(dstPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- archive.pipe(dstStream)+ const pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream+for (const f of dto.files) {if (await isPathIsDir(f.path)) {archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)} else {- archive.file(f.path, {- name: f.rootAlias ? f.name : fileName(f.path)- })+ archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })}}- await archive.finalize()++ const finalizePromise = archive.finalize()+ await Promise.all([finalizePromise, pipePromise])} finally {if (fileLock) {await this.filesLockManager.removeLock(fileLock.key)