Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): harden archive extraction and clean up partial output
johaven9615ed0ad238
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex d02742fe..e0cf7011 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -6,7 +6,6 @@ import { lookup } from 'node:dns/promises'import fs from 'node:fs'import path from 'node:path'import { PassThrough, Readable } from 'node:stream'-import * as tar from 'tar'import { transformAndValidate } from '../../../common/functions'import * as imageUtils from '../../../common/image'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'@@ -24,6 +23,7 @@ import { LockConflict } from '../models/file-lock-error'import { FILE_ERROR_MESSAGES } from '../utils/errors'import { SendFile } from '../utils/send-file'import * as unzipUtils from '../utils/unzip-file'+import * as untarUtils from '../utils/untar-file'import * as filesUtils from '../utils/files'import { FilesLockManager } from './files-lock-manager.service'import { FilesManager } from './files-manager.service'@@ -34,10 +34,6 @@ jest.mock('archiver', () => ({default: jest.fn()}))-jest.mock('tar', () => ({- __esModule: true,- extract: jest.fn()-}))jest.mock('node:dns/promises', () => ({lookup: jest.fn()}))@@ -994,21 +990,35 @@ describe(FilesManager.name, () => {expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')})- it('should extract tar formats via tar.extract', async () => {+ it('should extract tar formats via extractTar', async () => {const space = makeSpace({ realPath: '/data/users/john/files/archive.tar.gz' });(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true);(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')- ;(tar.extract as unknown as jest.Mock).mockResolvedValueOnce(undefined)+ const untarSpy = jest.spyOn(untarUtils, 'extractTar').mockResolvedValueOnce(undefined)await service.decompress(user, space)- expect(tar.extract).toHaveBeenCalledWith(- expect.objectContaining({- file: '/data/users/john/files/archive.tar.gz',- cwd: '/data/users/john/files/archive',- gzip: true- })- )+ expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/files/archive', true)+ })++ it('should remove partial extraction and skip add event on failure', async () => {+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ const error = new Error('extraction failed')+ jest.spyOn(unzipUtils, 'extractZip').mockRejectedValueOnce(error)+ const emitSpy = jest.spyOn(FileEvent, 'emit')++ await expect(service.decompress(user, space)).rejects.toBe(error)++ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/files/archive')+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')+ expect(emitSpy).not.toHaveBeenCalledWith('event', {+ user,+ space,+ action: ACTION.ADD,+ rPath: '/data/users/john/files/archive'+ })})})diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex c8dfd0a5..5268dc80 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -4,7 +4,6 @@ import archiver, { Archiver } from 'archiver'import fs from 'node:fs'import path from 'node:path'import { Readable } from 'node:stream'-import { extract as extractTar } from 'tar'import { FastifyAuthenticatedRequest } from '../../../authentication/interfaces/auth-request.interface'import { generateThumbnail } from '../../../common/image'import { SERVER_NAME } from '../../../common/shared'@@ -56,6 +55,7 @@ import {} from '../utils/files'import { SendFile } from '../utils/send-file'import { extractZip } from '../utils/unzip-file'+import { extractTar } from '../utils/untar-file'import { DownloadFile } from '../utils/download-file'import { FilesLockManager } from './files-lock-manager.service'import { FilesQueries } from './files-queries.service'@@ -607,7 +607,7 @@ export class FilesManager {}})// create lock- let fileLock: FileLock+ let fileLock: FileLock | undefinedif (dto.compressInDirectory) {const dbFile = space.dbFiledbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))@@ -658,32 +658,32 @@ export class FilesManager {// make destination folderconst dstPath = await uniqueFilePathFromDir(path.join(dirName(space.realPath), path.basename(space.realPath, extension)))await makeDir(dstPath)- // create lock- const dbFile = space.dbFile- dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))- const [ok, fileLock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.INFINITY)- if (!ok) {- throw new LockConflict(fileLock, 'Conflicting lock')- }- // tasking- if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath)- // do+ let fileLock: FileLock | undefinedtry {+ // create lock+ const dbFile = space.dbFile+ dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))+ const [ok, lock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.INFINITY)+ if (!ok) {+ throw new LockConflict(lock, 'Conflicting lock')+ }+ fileLock = lock+ // tasking+ if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath)+ // doif (extension === '.zip') {await extractZip(space.realPath, dstPath)} else {- await extractTar({- file: space.realPath,- cwd: dstPath,- gzip: COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION,- preserveOwner: false- })+ await extractTar(space.realPath, dstPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)}+ } catch (e) {+ await removeFiles(dstPath).catch((err: Error) => this.logger.error({ tag: this.decompress.name, msg: `unable to remove ${dstPath} : ${err}` }))+ throw e} finally {- await this.filesLockManager.removeLock(fileLock.key)- // emit file event- FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })+ if (fileLock) await this.filesLockManager.removeLock(fileLock.key)}+ // emit file event+ FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })}async generateThumbnail(space: SpaceEnv, size: number): Promise<Readable> {diff --git a/backend/src/applications/files/utils/untar-file.spec.ts b/backend/src/applications/files/utils/untar-file.spec.tsnew file mode 100644index 00000000..58f985dd--- /dev/null+++ b/backend/src/applications/files/utils/untar-file.spec.ts@@ -0,0 +1,61 @@+import { access, mkdir, mkdtemp, readlink, rm, symlink } from 'node:fs/promises'+import os from 'node:os'+import path from 'node:path'+import { create } from 'tar'+import { checkTarEntry, extractTar } from './untar-file'++describe(extractTar.name, () => {+ const baseOutputDir = path.join(path.sep, 'tmp', 'output')++ it('rejects hard links', () => {+ expect(() => checkTarEntry(baseOutputDir, { type: 'Link', path: 'docs/hard-link', linkpath: 'docs/target' })).toThrow(+ 'Tar entry "docs/hard-link" is a hard link'+ )+ })++ it('rejects symbolic links escaping the output directory', () => {+ expect(() => checkTarEntry(baseOutputDir, { type: 'SymbolicLink', path: 'docs/latest', linkpath: '../../../outside' })).toThrow(+ 'Tar symlink entry "docs/latest" would escape the output directory'+ )+ })++ let tmpDir: string+ let sourceDir: string+ let outputDir: string+ let archivePath: string++ beforeEach(async () => {+ tmpDir = await mkdtemp(path.join(os.tmpdir(), 'extract-tar-'))+ sourceDir = path.join(tmpDir, 'source')+ outputDir = path.join(tmpDir, 'output')+ archivePath = path.join(tmpDir, 'archive.tar')+ await mkdir(path.join(sourceDir, 'docs', 'v2'), { recursive: true })+ await mkdir(outputDir)+ })++ afterEach(async () => {+ await rm(tmpDir, { recursive: true, force: true })+ })++ it('extracts symbolic links targeting the output directory', async () => {+ await symlink('./v2', path.join(sourceDir, 'docs', 'latest'))+ await createArchive()++ await extractTar(archivePath, outputDir, false)++ await expect(readlink(path.join(outputDir, 'docs', 'latest'))).resolves.toBe('./v2')+ })++ it('rejects symbolic links escaping the output directory', async () => {+ const linkPath = path.join(outputDir, 'docs', 'latest')+ await symlink('../../../outside', path.join(sourceDir, 'docs', 'latest'))+ await createArchive()++ await expect(extractTar(archivePath, outputDir, false)).rejects.toThrow('Tar symlink entry "docs/latest" would escape the output directory')+ await expect(access(linkPath)).rejects.toMatchObject({ code: 'ENOENT' })+ })++ function createArchive(): Promise<void> {+ return create({ cwd: sourceDir, file: archivePath }, ['docs'])+ }+})diff --git a/backend/src/applications/files/utils/untar-file.ts b/backend/src/applications/files/utils/untar-file.tsnew file mode 100644index 00000000..f2e84eb9--- /dev/null+++ b/backend/src/applications/files/utils/untar-file.ts@@ -0,0 +1,42 @@+import path from 'node:path'+import { extract, type ReadEntry } from 'tar'+import { isPathInside } from './files'++export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' | 'path' | 'linkpath'>): boolean {+ if (entry.type === 'Link') {+ throw new Error(`Tar entry "${entry.path}" is a hard link`)+ }+ if (entry.type !== 'SymbolicLink') return true+ if (!entry.linkpath) {+ throw new Error(`Tar symlink entry "${entry.path}" has no target`)+ }+ const linkPath = path.resolve(outputDir, entry.path)+ const targetPath = path.resolve(path.dirname(linkPath), entry.linkpath)+ if (!isPathInside(outputDir, linkPath) || !isPathInside(outputDir, targetPath, true)) {+ throw new Error(`Tar symlink entry "${entry.path}" would escape the output directory`)+ }+ return true+}++export async function extractTar(filePath: string, outputDir: string, gzip: boolean): Promise<void> {+ // Do not throw from the stream filter callback: reject explicitly once parsing has completed.+ let validationError: Error | undefined+ await extract({+ file: filePath,+ cwd: outputDir,+ gzip,+ preserveOwner: false,+ preservePaths: false,+ strict: true,+ filter: (_path, entry) => {+ if (validationError) return false+ try {+ return !('type' in entry) || checkTarEntry(outputDir, entry)+ } catch (e) {+ validationError = e as Error+ return false+ }+ }+ })+ if (validationError) throw validationError+}