Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

refactor(backend:files): extract downloadFile and centralize SSRF, content-length, and quota checks

johavena2f86e17654b
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index aaaa988b..ff6373e1 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -6,6 +6,7 @@ import fs from 'node:fs'
import path from 'node:path'
import { PassThrough, Readable } from 'node:stream'
import * as tar from 'tar'
+import { transformAndValidate } from '../../../common/functions'
import * as imageUtils from '../../../common/image'
import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'
import { NotificationsManager } from '../../notifications/services/notifications-manager.service'
@@ -15,6 +16,7 @@ import * as spacesPermsUtils from '../../spaces/utils/permissions'
import { DEPTH } from '../../webdav/constants/webdav'
import { ACTION } from '../../../common/constants'
import { FILE_OPERATION } from '../constants/operations'
+import { DownloadFileDto } from '../dto/file-operations.dto'
import { FileEvent, FileTaskEvent } from '../events/file-events'
import { FileError } from '../models/file-error'
import { LockConflict } from '../models/file-lock-error'
@@ -350,11 +352,25 @@ describe(FilesManager.name, () => {
expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, true)
})
+ describe('downloadFromUrl dto validation', () => {
+ it('should accept http and https schemes', () => {
+ expect(transformAndValidate(DownloadFileDto, { url: 'https://example.org/file.txt' }).url).toBe('https://example.org/file.txt')
+ expect(transformAndValidate(DownloadFileDto, { url: 'http://example.org/file.txt' }).url).toBe('http://example.org/file.txt')
+ })
+
+ it('should reject non-http(s) schemes', () => {
+ const invalidUrls = ['ftp://example.org/file.txt', 'file:///tmp/file.txt', 'ws://example.org/file.txt']
+ for (const url of invalidUrls) {
+ expect(() => transformAndValidate(DownloadFileDto, { url })).toThrow()
+ }
+ })
+ })
+
it('downloadFromUrl should throw conflict when lock cannot be created', async () => {
const space = makeSpace()
filesLockManager.create.mockResolvedValueOnce([false, { key: 'other', owner: { id: 99 } }])
- await expect(service.downloadFromUrl(user, space, 'https://example.org/file.txt')).rejects.toBeInstanceOf(LockConflict)
+ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBeInstanceOf(LockConflict)
})
it('downloadFromUrl should handle HEAD+GET and emit task watch/event', async () => {
@@ -372,7 +388,7 @@ describe(FilesManager.name, () => {
const taskEmitSpy = jest.spyOn(FileTaskEvent, 'emit')
const fileEmitSpy = jest.spyOn(FileEvent, 'emit')
- await service.downloadFromUrl(user, space, 'https://example.org/file.txt')
+ await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })
expect(space.task.props.totalSize).toBe(55)
expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DOWNLOAD, '/tmp/download.txt')
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 7f7677b5..51fc10c2 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -1,7 +1,6 @@
import { HttpService } from '@nestjs/axios'
import { HttpStatus, Injectable, Logger } from '@nestjs/common'
import archiver, { Archiver } from 'archiver'
-import { AxiosResponse } from 'axios'
import fs from 'node:fs'
import path from 'node:path'
import { Readable } from 'node:stream'
@@ -27,7 +26,7 @@ import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
import { FILE_OPERATION } from '../constants/operations'
import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
-import { CompressFileDto } from '../dto/file-operations.dto'
+import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
import { FileDBProps } from '../interfaces/file-db-props.interface'
import { FileLock } from '../interfaces/file-lock.interface'
import { FileLockProps } from '../interfaces/file-props.interface'
@@ -56,7 +55,7 @@ import {
} from '../utils/files'
import { SendFile } from '../utils/send-file'
import { extractZip } from '../utils/unzip-file'
-import { regExpPrivateIP } from '../utils/url-file'
+import { downloadFile } from '../utils/download-file'
import { FilesLockManager } from './files-lock-manager.service'
import { FilesQueries } from './files-queries.service'
import { FileEvent, FileTaskEvent } from '../events/file-events'
@@ -503,54 +502,21 @@ export class FilesManager {
await this.filesQueries.deleteFiles(space.dbFile, isDir, forceDeleteInDB)
}
- async downloadFromUrl(user: UserModel, space: SpaceEnv, url: string): Promise<void> {
- this.logger.log({ tag: this.downloadFromUrl.name, msg: `${url}` })
- // create lock
+ async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {
+ this.logger.log({ tag: this.downloadFromUrl.name, msg: `${downloadDto.url}` })
const rPath = await uniqueFilePathFromDir(space.realPath)
const dbFile = space.dbFile
dbFile.path = path.join(dirName(dbFile.path), fileName(space.realPath))
+
+ // create lock
const [ok, fileLock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.RESOURCE)
if (!ok) {
throw new LockConflict(fileLock, 'Conflicting lock')
}
- // tasking
- if (space.task.cacheKey) {
- let headRes: AxiosResponse
-
- try {
- headRes = await this.http.axiosRef({ method: HTTP_METHOD.HEAD, url: url, maxRedirects: 1 })
- } catch (e) {
- // release lock
- await this.filesLockManager.removeLock(fileLock.key)
- this.logger.error({ tag: this.downloadFromUrl.name, msg: `${url} : ${e}` })
- throw new FileError(HttpStatus.BAD_REQUEST, 'Unable to download file')
- }
- if (regExpPrivateIP.test(headRes.request.socket.remoteAddress)) {
- // release lock
- await this.filesLockManager.removeLock(fileLock.key)
- // prevent SSRF attack
- throw new FileError(HttpStatus.FORBIDDEN, 'Access to internal IP addresses is forbidden')
- }
-
- // attempt to retrieve the Content-Length header
- try {
- if ('content-length' in headRes.headers) {
- space.task.props.totalSize = Number(headRes.headers['content-length']) || null
- }
- } catch (e) {
- this.logger.debug({ tag: this.downloadFromUrl.name, msg: `content-length : ${e}` })
- }
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, rPath)
- }
// do
try {
- const getRes = await this.http.axiosRef({ method: HTTP_METHOD.GET, url: url, responseType: 'stream', maxRedirects: 1 })
- if (regExpPrivateIP.test(getRes.request.socket.remoteAddress)) {
- // Prevent SSRF attacks and perform a DNS-rebinding check if a HEAD request has already been made
- throw new FileError(HttpStatus.FORBIDDEN, 'Access to internal IP addresses is forbidden')
- }
- await writeFromStream(rPath, getRes.data)
+ await downloadFile(this.http, downloadDto, rPath, space)
} finally {
// release lock
await this.filesLockManager.removeLock(fileLock.key)
diff --git a/backend/src/applications/files/services/files-methods.service.ts b/backend/src/applications/files/services/files-methods.service.ts
index f534a6b0..c4b4e512 100644
--- a/backend/src/applications/files/services/files-methods.service.ts
+++ b/backend/src/applications/files/services/files-methods.service.ts
@@ -87,7 +87,7 @@ export class FilesMethods {
async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {
checkFileName(space.realPath)
try {
- return await this.filesManager.downloadFromUrl(user, space, downloadDto.url)
+ return await this.filesManager.downloadFromUrl(user, space, downloadDto)
} catch (e) {
this.handleError(space, FILE_OPERATION.DOWNLOAD, e)
}
diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.ts
new file mode 100644
index 00000000..8f66003b
--- /dev/null
+++ b/backend/src/applications/files/utils/download-file.ts
@@ -0,0 +1,71 @@
+import type { HttpService } from '@nestjs/axios'
+import type { SpaceEnv } from '../../spaces/models/space-env.model'
+import type { AxiosResponse } from 'axios'
+import { HTTP_METHOD } from '../../applications.constants'
+import { FileError } from '../models/file-error'
+import { HttpStatus } from '@nestjs/common'
+import { FileTaskEvent } from '../events/file-events'
+import { FILE_OPERATION } from '../constants/operations'
+import { writeFromStream } from './files'
+import { DownloadFileDto } from '../dto/file-operations.dto'
+
+const parts = [
+ // IPv4 loopback (127.0.0.0/8)
+ '127\\.(?:\\d{1,3}\\.){2}\\d{1,3}',
+ // IPv4 link-local (169.254.0.0/16)
+ '169\\.254\\.\\d{1,3}\\.\\d{1,3}',
+ // IPv4 Carrier-grade NAT (100.64.0.0/10)
+ '100\\.(?:6[4-9]|[7-9]\\d|1[01]\\d|12[0-7])\\.\\d{1,3}\\.\\d{1,3}',
+ // IPv4 private (10.0.0.0/8)
+ '10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}',
+ // IPv4 private (192.168.0.0/16)
+ '192\\.168\\.\\d{1,3}\\.\\d{1,3}',
+ // IPv4 private (172.16.0.0/12)
+ '172\\.(?:1[6-9]|2\\d|3[0-1])\\.\\d{1,3}\\.\\d{1,3}',
+ // IPv4 & IPv6 loopback
+ '::1',
+ '::',
+ '0.0.0.0',
+ // IPv6 Unique Local Address (fc00::/7)
+ 'f[cd][0-9a-f]{2}:[0-9a-f:]+',
+ // IPv6 link-local (fe80::/10)
+ 'fe[89ab][0-9a-f]{2}:[0-9a-f:]+'
+]
+
+const regExpPrivateIP = new RegExp(`^(?:${parts.join('|')})$`, 'i')
+const errorRegexpPrivateIP = 'Access to internal IP addresses is forbidden'
+
+export async function downloadFile(http: HttpService, downloadDto: DownloadFileDto, dstPath: string, space?: SpaceEnv) {
+ // dto must be validated by the caller
+ const headRes: AxiosResponse = await http.axiosRef({ method: HTTP_METHOD.HEAD, url: downloadDto.url, maxRedirects: 1 })
+ if (regExpPrivateIP.test(headRes.request.socket.remoteAddress)) {
+ // prevent SSRF attack
+ throw new FileError(HttpStatus.FORBIDDEN, errorRegexpPrivateIP)
+ }
+
+ // attempt to retrieve the Content-Length header
+ const contentLength = 'content-length' in headRes.headers ? Number(headRes.headers['content-length']) || null : null
+ if (!contentLength) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'Missing "content-length" header')
+ }
+
+ if (space) {
+ if (space.willExceedQuota(contentLength)) {
+ throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, 'Storage quota will be exceeded')
+ }
+ // tasking
+ if (space.task.cacheKey) {
+ space.task.props.totalSize = contentLength
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)
+ }
+ }
+
+ const getRes = await http.axiosRef({ method: HTTP_METHOD.GET, url: downloadDto.url, responseType: 'stream', maxRedirects: 1 })
+ if (regExpPrivateIP.test(getRes.request.socket.remoteAddress)) {
+ // close request
+ getRes.data?.destroy()
+ // Prevent SSRF attacks and perform a DNS-rebinding check if a HEAD request has already been made
+ throw new FileError(HttpStatus.FORBIDDEN, errorRegexpPrivateIP)
+ }
+ await writeFromStream(dstPath, getRes.data)
+}
diff --git a/backend/src/applications/files/utils/url-file.ts b/backend/src/applications/files/utils/url-file.ts
deleted file mode 100644
index f613a8f2..00000000
--- a/backend/src/applications/files/utils/url-file.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-const parts = [
- // IPv4 loopback (127.0.0.0/8)
- '127\\.(?:\\d{1,3}\\.){2}\\d{1,3}',
- // IPv4 link-local (169.254.0.0/16)
- '169\\.254\\.\\d{1,3}\\.\\d{1,3}',
- // IPv4 Carrier-grade NAT (100.64.0.0/10)
- '100\\.(?:6[4-9]|[7-9]\\d|1[01]\\d|12[0-7])\\.\\d{1,3}\\.\\d{1,3}',
- // IPv4 private (10.0.0.0/8)
- '10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}',
- // IPv4 private (192.168.0.0/16)
- '192\\.168\\.\\d{1,3}\\.\\d{1,3}',
- // IPv4 private (172.16.0.0/12)
- '172\\.(?:1[6-9]|2\\d|3[0-1])\\.\\d{1,3}\\.\\d{1,3}',
- // IPv4 & IPv6 loopback
- '::1',
- '::',
- '0.0.0.0',
- // IPv6 Unique Local Address (fc00::/7)
- 'f[cd][0-9a-f]{2}:[0-9a-f:]+',
- // IPv6 link-local (fe80::/10)
- 'fe[89ab][0-9a-f]{2}:[0-9a-f:]+'
-]
-
-export const regExpPrivateIP = new RegExp(`^(?:${parts.join('|')})$`, 'i')