Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): enforce storage quota during archive extraction

johaven8fffc17adf21
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 858dd74c..c104a213 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -1156,7 +1156,7 @@ describe(FilesManager.name, () => {
await service.decompress(user, space)
expect(filesUtils.makeTempDir).toHaveBeenCalledWith('/data/users/john/tmp', 'archive-extract-')
- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123')
+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', undefined)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
expect(taskEmitSpy).toHaveBeenCalledWith(
'startWatch',
@@ -1177,10 +1177,22 @@ describe(FilesManager.name, () => {
await service.decompress(user, space)
- expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true)
+ expect(untarSpy).toHaveBeenCalledWith('/data/users/john/files/archive.tar.gz', '/data/users/john/tmp/archive-extract-123', true, undefined)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
})
+ it('should limit extracted size to the known remaining quota', async () => {
+ const space = makeSpace({ realPath: '/data/users/john/files/archive.zip', storageQuota: 100, storageUsage: 40 })
+ ;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false)
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/data/users/john/files/archive')
+ ;(filesUtils.makeTempDir as jest.Mock).mockResolvedValueOnce('/data/users/john/tmp/archive-extract-123')
+ const unzipSpy = jest.spyOn(unzipUtils, 'extractZip').mockResolvedValueOnce(undefined)
+
+ await service.decompress(user, space)
+
+ expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', 60)
+ })
+
it('should remove partial extraction and skip add event on failure', async () => {
const space = makeSpace({ realPath: '/data/users/john/files/archive.zip' })
;(filesUtils.isPathExists as jest.Mock).mockResolvedValueOnce(true)
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 5f3f691e..9d72df2f 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -694,10 +694,11 @@ export class FilesManager {
// tasking
if (space.task?.cacheKey) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
// do
+ const maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)
if (extension === '.zip') {
- await extractZip(space.realPath, tmpPath)
+ await extractZip(space.realPath, tmpPath, maxExtractedSize)
} else {
- await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION)
+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize)
}
if (await isPathExists(dstPath)) {
throw new FileError(HttpStatus.CONFLICT, 'The destination already exists')
diff --git a/backend/src/applications/files/utils/errors.ts b/backend/src/applications/files/utils/errors.ts
index 220106a7..e11aa651 100644
--- a/backend/src/applications/files/utils/errors.ts
+++ b/backend/src/applications/files/utils/errors.ts
@@ -14,3 +14,7 @@ export const FILE_ERROR_MESSAGES = {
export function maxFileSizeExceededError(): FileError {
return new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR_MESSAGES.MAX_FILE_SIZE_EXCEEDED)
}
+
+export function storageQuotaExceededError(): FileError {
+ return new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)
+}
diff --git a/backend/src/applications/files/utils/untar-file.spec.ts b/backend/src/applications/files/utils/untar-file.spec.ts
index 58f985dd..fb4f53d3 100644
--- a/backend/src/applications/files/utils/untar-file.spec.ts
+++ b/backend/src/applications/files/utils/untar-file.spec.ts
@@ -1,4 +1,5 @@
-import { access, mkdir, mkdtemp, readlink, rm, symlink } from 'node:fs/promises'
+import fs from 'node:fs'
+import { access, mkdir, mkdtemp, readlink, rm, symlink, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { create } from 'tar'
@@ -34,6 +35,7 @@ describe(extractTar.name, () => {
})
afterEach(async () => {
+ jest.restoreAllMocks()
await rm(tmpDir, { recursive: true, force: true })
})
@@ -55,7 +57,26 @@ describe(extractTar.name, () => {
await expect(access(linkPath)).rejects.toMatchObject({ code: 'ENOENT' })
})
- function createArchive(): Promise<void> {
- return create({ cwd: sourceDir, file: archivePath }, ['docs'])
+ it('rejects entries exceeding the extracted size limit', async () => {
+ await writeFile(path.join(sourceDir, 'docs', 'large.txt'), 'ab')
+ await createArchive()
+ const destroySpy = jest.spyOn(fs.ReadStream.prototype, 'destroy')
+
+ await expect(extractTar(archivePath, outputDir, false, 1)).rejects.toThrow('Storage quota will be exceeded')
+ expect(destroySpy).toHaveBeenCalled()
+ })
+
+ it('aborts TAR.GZ extraction when the extracted size limit is exceeded', async () => {
+ archivePath = path.join(tmpDir, 'archive.tar.gz')
+ await writeFile(path.join(sourceDir, 'docs', 'large.txt'), 'ab')
+ await createArchive(true)
+ const destroySpy = jest.spyOn(fs.ReadStream.prototype, 'destroy')
+
+ await expect(extractTar(archivePath, outputDir, true, 1)).rejects.toThrow('Storage quota will be exceeded')
+ expect(destroySpy).toHaveBeenCalled()
+ })
+
+ function createArchive(gzip = false): Promise<void> {
+ return create({ cwd: sourceDir, file: archivePath, gzip }, ['docs'])
}
})
diff --git a/backend/src/applications/files/utils/untar-file.ts b/backend/src/applications/files/utils/untar-file.ts
index f2e84eb9..cb92f635 100644
--- a/backend/src/applications/files/utils/untar-file.ts
+++ b/backend/src/applications/files/utils/untar-file.ts
@@ -1,5 +1,8 @@
+import { createReadStream } from 'node:fs'
import path from 'node:path'
-import { extract, type ReadEntry } from 'tar'
+import { pipeline } from 'node:stream/promises'
+import { extract, type ReadEntry, type Unpack } from 'tar'
+import { storageQuotaExceededError } from './errors'
import { isPathInside } from './files'
export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' | 'path' | 'linkpath'>): boolean {
@@ -18,11 +21,20 @@ export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' |
return true
}
-export async function extractTar(filePath: string, outputDir: string, gzip: boolean): Promise<void> {
- // Do not throw from the stream filter callback: reject explicitly once parsing has completed.
+export async function extractTar(filePath: string, outputDir: string, gzip: boolean, maxExtractedSize?: number): Promise<void> {
let validationError: Error | undefined
- await extract({
- file: filePath,
+ let extractedSize = 0
+ const srcStream = createReadStream(filePath)
+
+ const abortExtraction = (error: Error): false => {
+ // Stop both the file read and node-tar parser so a rejected TAR.GZ is not decompressed to the end.
+ validationError = error
+ srcStream.destroy(error)
+ extractStream.abort(error)
+ return false
+ }
+
+ const extractStream: Unpack = extract({
cwd: outputDir,
gzip,
preserveOwner: false,
@@ -31,12 +43,20 @@ export async function extractTar(filePath: string, outputDir: string, gzip: bool
filter: (_path, entry) => {
if (validationError) return false
try {
+ // node-tar invokes the filter before writing an entry, so metadata can enforce the known quota.
+ extractedSize += entry.size
+ if (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {
+ throw storageQuotaExceededError()
+ }
return !('type' in entry) || checkTarEntry(outputDir, entry)
} catch (e) {
- validationError = e as Error
- return false
+ return abortExtraction(e as Error)
}
}
})
- if (validationError) throw validationError
+ try {
+ await pipeline(srcStream, extractStream)
+ } catch (e) {
+ throw validationError || e
+ }
}
diff --git a/backend/src/applications/files/utils/unzip-file.spec.ts b/backend/src/applications/files/utils/unzip-file.spec.ts
index 0df914e7..22e1da1d 100644
--- a/backend/src/applications/files/utils/unzip-file.spec.ts
+++ b/backend/src/applications/files/utils/unzip-file.spec.ts
@@ -4,17 +4,21 @@ import path from 'node:path'
import * as filesUtils from './files'
import { extractZip } from './unzip-file'
-function createEmptyZip(entryName: string): Buffer {
+function createZip(entryName: string, content = Buffer.alloc(0)): Buffer {
const encodedEntryName = Buffer.from(entryName)
const localFileHeader = Buffer.alloc(30)
localFileHeader.writeUInt32LE(0x04034b50, 0)
localFileHeader.writeUInt16LE(20, 4)
+ localFileHeader.writeUInt32LE(content.length, 18)
+ localFileHeader.writeUInt32LE(content.length, 22)
localFileHeader.writeUInt16LE(encodedEntryName.length, 26)
const centralDirectoryHeader = Buffer.alloc(46)
centralDirectoryHeader.writeUInt32LE(0x02014b50, 0)
centralDirectoryHeader.writeUInt16LE(20, 4)
centralDirectoryHeader.writeUInt16LE(20, 6)
+ centralDirectoryHeader.writeUInt32LE(content.length, 20)
+ centralDirectoryHeader.writeUInt32LE(content.length, 24)
centralDirectoryHeader.writeUInt16LE(encodedEntryName.length, 28)
const endOfCentralDirectory = Buffer.alloc(22)
@@ -22,9 +26,9 @@ function createEmptyZip(entryName: string): Buffer {
endOfCentralDirectory.writeUInt16LE(1, 8)
endOfCentralDirectory.writeUInt16LE(1, 10)
endOfCentralDirectory.writeUInt32LE(centralDirectoryHeader.length + encodedEntryName.length, 12)
- endOfCentralDirectory.writeUInt32LE(localFileHeader.length + encodedEntryName.length, 16)
+ endOfCentralDirectory.writeUInt32LE(localFileHeader.length + encodedEntryName.length + content.length, 16)
- return Buffer.concat([localFileHeader, encodedEntryName, centralDirectoryHeader, encodedEntryName, endOfCentralDirectory])
+ return Buffer.concat([localFileHeader, encodedEntryName, content, centralDirectoryHeader, encodedEntryName, endOfCentralDirectory])
}
describe(extractZip.name, () => {
@@ -46,7 +50,7 @@ describe(extractZip.name, () => {
it('rejects entries escaping the output directory', async () => {
const escapedPath = path.join(tmpDir, 'zip-slip-proof.txt')
- await writeFile(archivePath, createEmptyZip('../zip-slip-proof.txt'))
+ await writeFile(archivePath, createZip('../zip-slip-proof.txt'))
await expect(extractZip(archivePath, outputDir)).rejects.toThrow('invalid relative path: ../zip-slip-proof.txt')
await expect(access(escapedPath)).rejects.toMatchObject({ code: 'ENOENT' })
@@ -54,14 +58,14 @@ describe(extractZip.name, () => {
it('extracts entries inside the output directory', async () => {
const extractedPath = path.join(outputDir, 'safe.txt')
- await writeFile(archivePath, createEmptyZip('safe.txt'))
+ await writeFile(archivePath, createZip('safe.txt'))
await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()
await expect(access(extractedPath)).resolves.toBeUndefined()
})
it('rejects entries outside the output directory', async () => {
- await writeFile(archivePath, createEmptyZip('safe.txt'))
+ await writeFile(archivePath, createZip('safe.txt'))
const isPathInsideSpy = jest.spyOn(filesUtils, 'isPathInside').mockReturnValueOnce(false)
await expect(extractZip(archivePath, outputDir)).rejects.toThrow('Zip entry "safe.txt" would escape the output directory')
@@ -69,10 +73,16 @@ describe(extractZip.name, () => {
})
it('accepts a root directory entry', async () => {
- await writeFile(archivePath, createEmptyZip('./'))
+ await writeFile(archivePath, createZip('./'))
const isPathInsideSpy = jest.spyOn(filesUtils, 'isPathInside')
await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()
expect(isPathInsideSpy).toHaveBeenCalledWith(outputDir, outputDir, true)
})
+
+ it('rejects entries exceeding the extracted size limit', async () => {
+ await writeFile(archivePath, createZip('large.txt', Buffer.from('ab')))
+
+ await expect(extractZip(archivePath, outputDir, 1)).rejects.toThrow('Storage quota will be exceeded')
+ })
})
diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.ts
index 12e0a3a3..e964458f 100644
--- a/backend/src/applications/files/utils/unzip-file.ts
+++ b/backend/src/applications/files/utils/unzip-file.ts
@@ -4,18 +4,26 @@ import { pipeline } from 'node:stream/promises'
import { promisify } from 'node:util'
import { Entry, open as openZip, Options, ZipFile } from 'yauzl'
import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'
+import { storageQuotaExceededError } from './errors'
import { isPathInside, makeDir } from './files'
const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)
-export async function extractZip(filePath: string, outputDir: string): Promise<void> {
+export async function extractZip(filePath: string, outputDir: string, maxExtractedSize?: number): Promise<void> {
+ // Reject entries whose actual decompressed size differs from their metadata.
const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })
const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))
const resolvedOutputDir = path.resolve(outputDir)
+ let extractedSize = 0
return new Promise((resolve, reject) => {
zipFile.on('entry', async (entry: Entry) => {
try {
+ // Check the cumulative size before opening the entry stream to avoid writing beyond the known quota.
+ extractedSize += entry.uncompressedSize
+ if (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {
+ throw storageQuotaExceededError()
+ }
const isDir = entry.fileName.endsWith('/')
const fullPath = path.resolve(resolvedOutputDir, entry.fileName)
if (!isPathInside(resolvedOutputDir, fullPath, isDir)) {