Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): enforce storage quota during archive extraction
johaven8fffc17adf21
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 858dd74c..c104a213 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -1156,7 +1156,7 @@ describe(FilesManager.name, () => {await service.decompress(user, space)expect(filesUtils.makeTempDir).toHaveBeenCalledWith('/data/users/john/tmp', 'archive-extract-')- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123')+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', undefined)expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')expect(taskEmitSpy).toHaveBeenCalledWith('startWatch',@@ -1177,10 +1177,22 @@ describe(FilesManager.name, () => {await service.decompress(user, space)- expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true)+ expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true, undefined)expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')})+ it('should limit extracted size to the known remaining quota', async () => {+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip', storageQuota: 100, storageUsage: 40 })+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')+ const unzipSpy = jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)++ await service.decompress(user, space)++ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', 60)+ })+it('should remove partial extraction and skip add event on failure', async () => {const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' });(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 5f3f691e..9d72df2f 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -694,10 +694,11 @@ export class FilesManager {// taskingif (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)// do+ const maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)if (extension === '.zip') {- await extractZip(space.realPath, tmpPath)+ await extractZip(space.realPath, tmpPath, maxExtractedSize)} else {- await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize)}if (await isPathExists(dstPath)) {throw new FileError(HttpStatus.CONFLICT, 'The destination already exists')diff --git a/backend/src/applications/files/utils/errors.ts b/backend/src/applications/files/utils/errors.tsindex 220106a7..e11aa651 100644--- a/backend/src/applications/files/utils/errors.ts+++ b/backend/src/applications/files/utils/errors.ts@@ -14,3 +14,7 @@ export const FILE_ERROR_MESSAGES = {export function maxFileSizeExceededError(): FileError {return new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR_MESSAGES.MAX_FILE_SIZE_EXCEEDED)}++export function storageQuotaExceededError(): FileError {+ return new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)+}diff --git a/backend/src/applications/files/utils/untar-file.spec.ts b/backend/src/applications/files/utils/untar-file.spec.tsindex 58f985dd..fb4f53d3 100644--- a/backend/src/applications/files/utils/untar-file.spec.ts+++ b/backend/src/applications/files/utils/untar-file.spec.ts@@ -1,4 +1,5 @@-import { access, mkdir, mkdtemp, readlink, rm, symlink } from 'node:fs/promises'+import fs from 'node:fs'+import { access, mkdir, mkdtemp, readlink, rm, symlink, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { create } from 'tar'@@ -34,6 +35,7 @@ describe(extractTar.name, () => {})afterEach(async () => {+ jest.restoreAllMocks()await rm(tmpDir, { recursive: true, force: true })})@@ -55,7 +57,26 @@ describe(extractTar.name, () => {await expect(access(linkPath)).rejects.toMatchObject({ code: 'ENOENT' })})- function createArchive(): Promise<void> {- return create({ cwd: sourceDir, file: archivePath }, ['docs'])+ it('rejects entries exceeding the extracted size limit', async () => {+ await writeFile(path.join(sourceDir, 'docs', 'large.txt'), 'ab')+ await createArchive()+ const destroySpy = jest.spyOn(fs.ReadStream.prototype, 'destroy')++ await expect(extractTar(archivePath, outputDir, false, 1)).rejects.toThrow('Storage quota will be exceeded')+ expect(destroySpy).toHaveBeenCalled()+ })++ it('aborts TAR.GZ extraction when the extracted size limit is exceeded', async () => {+ archivePath = path.join(tmpDir, 'archive.tar.gz')+ await writeFile(path.join(sourceDir, 'docs', 'large.txt'), 'ab')+ await createArchive(true)+ const destroySpy = jest.spyOn(fs.ReadStream.prototype, 'destroy')++ await expect(extractTar(archivePath, outputDir, true, 1)).rejects.toThrow('Storage quota will be exceeded')+ expect(destroySpy).toHaveBeenCalled()+ })++ function createArchive(gzip = false): Promise<void> {+ return create({ cwd: sourceDir, file: archivePath, gzip }, ['docs'])}})diff --git a/backend/src/applications/files/utils/untar-file.ts b/backend/src/applications/files/utils/untar-file.tsindex f2e84eb9..cb92f635 100644--- a/backend/src/applications/files/utils/untar-file.ts+++ b/backend/src/applications/files/utils/untar-file.ts@@ -1,5 +1,8 @@+import { createReadStream } from 'node:fs'import path from 'node:path'-import { extract, type ReadEntry } from 'tar'+import { pipeline } from 'node:stream/promises'+import { extract, type ReadEntry, type Unpack } from 'tar'+import { storageQuotaExceededError } from './errors'import { isPathInside } from './files'export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' | 'path' | 'linkpath'>): boolean {@@ -18,11 +21,20 @@ export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' |return true}-export async function extractTar(filePath: string, outputDir: string, gzip: boolean): Promise<void> {- // Do not throw from the stream filter callback: reject explicitly once parsing has completed.+export async function extractTar(filePath: string, outputDir: string, gzip: boolean, maxExtractedSize?: number): Promise<void> {let validationError: Error | undefined- await extract({- file: filePath,+ let extractedSize = 0+ const srcStream = createReadStream(filePath)++ const abortExtraction = (error: Error): false => {+ // Stop both the file read and node-tar parser so a rejected TAR.GZ is not decompressed to the end.+ validationError = error+ srcStream.destroy(error)+ extractStream.abort(error)+ return false+ }++ const extractStream: Unpack = extract({cwd: outputDir,gzip,preserveOwner: false,@@ -31,12 +43,20 @@ export async function extractTar(filePath: string, outputDir: string, gzip: boolfilter: (_path, entry) => {if (validationError) return falsetry {+ // node-tar invokes the filter before writing an entry, so metadata can enforce the known quota.+ extractedSize += entry.size+ if (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {+ throw storageQuotaExceededError()+ }return !('type' in entry) || checkTarEntry(outputDir, entry)} catch (e) {- validationError = e as Error- return false+ return abortExtraction(e as Error)}}})- if (validationError) throw validationError+ try {+ await pipeline(srcStream, extractStream)+ } catch (e) {+ throw validationError || e+ }}diff --git a/backend/src/applications/files/utils/unzip-file.spec.ts b/backend/src/applications/files/utils/unzip-file.spec.tsindex 0df914e7..22e1da1d 100644--- a/backend/src/applications/files/utils/unzip-file.spec.ts+++ b/backend/src/applications/files/utils/unzip-file.spec.ts@@ -4,17 +4,21 @@ import path from 'node:path'import * as filesUtils from './files'import { extractZip } from './unzip-file'-function createEmptyZip(entryName: string): Buffer {+function createZip(entryName: string, content = Buffer.alloc(0)): Buffer {const encodedEntryName = Buffer.from(entryName)const localFileHeader = Buffer.alloc(30)localFileHeader.writeUInt32LE(0x04034b50, 0)localFileHeader.writeUInt16LE(20, 4)+ localFileHeader.writeUInt32LE(content.length, 18)+ localFileHeader.writeUInt32LE(content.length, 22)localFileHeader.writeUInt16LE(encodedEntryName.length, 26)const centralDirectoryHeader = Buffer.alloc(46)centralDirectoryHeader.writeUInt32LE(0x02014b50, 0)centralDirectoryHeader.writeUInt16LE(20, 4)centralDirectoryHeader.writeUInt16LE(20, 6)+ centralDirectoryHeader.writeUInt32LE(content.length, 20)+ centralDirectoryHeader.writeUInt32LE(content.length, 24)centralDirectoryHeader.writeUInt16LE(encodedEntryName.length, 28)const endOfCentralDirectory = Buffer.alloc(22)@@ -22,9 +26,9 @@ function createEmptyZip(entryName: string): Buffer {endOfCentralDirectory.writeUInt16LE(1, 8)endOfCentralDirectory.writeUInt16LE(1, 10)endOfCentralDirectory.writeUInt32LE(centralDirectoryHeader.length + encodedEntryName.length, 12)- endOfCentralDirectory.writeUInt32LE(localFileHeader.length + encodedEntryName.length, 16)+ endOfCentralDirectory.writeUInt32LE(localFileHeader.length + encodedEntryName.length + content.length, 16)- return Buffer.concat([localFileHeader, encodedEntryName, centralDirectoryHeader, encodedEntryName, endOfCentralDirectory])+ return Buffer.concat([localFileHeader, encodedEntryName, content, centralDirectoryHeader, encodedEntryName, endOfCentralDirectory])}describe(extractZip.name, () => {@@ -46,7 +50,7 @@ describe(extractZip.name, () => {it('rejects entries escaping the output directory', async () => {const escapedPath = path.join(tmpDir, 'zip-slip-proof.txt')- await writeFile(archivePath, createEmptyZip('../zip-slip-proof.txt'))+ await writeFile(archivePath, createZip('../zip-slip-proof.txt'))await expect(extractZip(archivePath, outputDir)).rejects.toThrow('invalid relative path: ../zip-slip-proof.txt')await expect(access(escapedPath)).rejects.toMatchObject({ code: 'ENOENT' })@@ -54,14 +58,14 @@ describe(extractZip.name, () => {it('extracts entries inside the output directory', async () => {const extractedPath = path.join(outputDir, 'safe.txt')- await writeFile(archivePath, createEmptyZip('safe.txt'))+ await writeFile(archivePath, createZip('safe.txt'))await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()await expect(access(extractedPath)).resolves.toBeUndefined()})it('rejects entries outside the output directory', async () => {- await writeFile(archivePath, createEmptyZip('safe.txt'))+ await writeFile(archivePath, createZip('safe.txt'))const isPathInsideSpy = jest.spyOn(filesUtils, 'isPathInside').mockReturnValueOnce(false)await expect(extractZip(archivePath, outputDir)).rejects.toThrow('Zip entry "safe.txt" would escape the output directory')@@ -69,10 +73,16 @@ describe(extractZip.name, () => {})it('accepts a root directory entry', async () => {- await writeFile(archivePath, createEmptyZip('./'))+ await writeFile(archivePath, createZip('./'))const isPathInsideSpy = jest.spyOn(filesUtils, 'isPathInside')await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()expect(isPathInsideSpy).toHaveBeenCalledWith(outputDir, outputDir, true)})++ it('rejects entries exceeding the extracted size limit', async () => {+ await writeFile(archivePath, createZip('large.txt', Buffer.from('ab')))++ await expect(extractZip(archivePath, outputDir, 1)).rejects.toThrow('Storage quota will be exceeded')+ })})diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.tsindex 12e0a3a3..e964458f 100644--- a/backend/src/applications/files/utils/unzip-file.ts+++ b/backend/src/applications/files/utils/unzip-file.ts@@ -4,18 +4,26 @@ import { pipeline } from 'node:stream/promises'import { promisify } from 'node:util'import { Entry, open as openZip, Options, ZipFile } from 'yauzl'import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'+import { storageQuotaExceededError } from './errors'import { isPathInside, makeDir } from './files'const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)-export async function extractZip(filePath: string, outputDir: string): Promise<void> {+export async function extractZip(filePath: string, outputDir: string, maxExtractedSize?: number): Promise<void> {+ // Reject entries whose actual decompressed size differs from their metadata.const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))const resolvedOutputDir = path.resolve(outputDir)+ let extractedSize = 0return new Promise((resolve, reject) => {zipFile.on('entry', async (entry: Entry) => {try {+ // Check the cumulative size before opening the entry stream to avoid writing beyond the known quota.+ extractedSize += entry.uncompressedSize+ if (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {+ throw storageQuotaExceededError()+ }const isDir = entry.fileName.endsWith('/')const fullPath = path.resolve(resolvedOutputDir, entry.fileName)if (!isPathInside(resolvedOutputDir, fullPath, isDir)) {