Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
refactor(backend:files): extract downloadFile and centralize SSRF, content-length, and quota checks
johavena2f86e17654b
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex aaaa988b..ff6373e1 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -6,6 +6,7 @@ import fs from 'node:fs'import path from 'node:path'import { PassThrough, Readable } from 'node:stream'import * as tar from 'tar'+import { transformAndValidate } from '../../../common/functions'import * as imageUtils from '../../../common/image'import { ContextManager } from '../../../infrastructure/context/services/context-manager.service'import { NotificationsManager } from '../../notifications/services/notifications-manager.service'@@ -15,6 +16,7 @@ import * as spacesPermsUtils from '../../spaces/utils/permissions'import { DEPTH } from '../../webdav/constants/webdav'import { ACTION } from '../../../common/constants'import { FILE_OPERATION } from '../constants/operations'+import { DownloadFileDto } from '../dto/file-operations.dto'import { FileEvent, FileTaskEvent } from '../events/file-events'import { FileError } from '../models/file-error'import { LockConflict } from '../models/file-lock-error'@@ -350,11 +352,25 @@ describe(FilesManager.name, () => {expect(filesQueries.deleteFiles).toHaveBeenCalledWith(space.dbFile, false, true)})+ describe('downloadFromUrl dto validation', () => {+ it('should accept http and https schemes', () => {+ expect(transformAndValidate(DownloadFileDto, { url: 'https://example.org/file.txt' }).url).toBe('https://example.org/file.txt')+ expect(transformAndValidate(DownloadFileDto, { url: 'http://example.org/file.txt' }).url).toBe('http://example.org/file.txt')+ })++ it('should reject non-http(s) schemes', () => {+ const invalidUrls = ['ftp://example.org/file.txt', 'file:///tmp/file.txt', 'ws://example.org/file.txt']+ for (const url of invalidUrls) {+ expect(() => transformAndValidate(DownloadFileDto, { url })).toThrow()+ }+ })+ })+it('downloadFromUrl should throw conflict when lock cannot be created', async () => {const space = makeSpace()filesLockManager.create.mockResolvedValueOnce([false, { key: 'other', owner: { id: 99 } }])- await expect(service.downloadFromUrl(user, space, 'https://example.org/file.txt')).rejects.toBeInstanceOf(LockConflict)+ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBeInstanceOf(LockConflict)})it('downloadFromUrl should handle HEAD+GET and emit task watch/event', async () => {@@ -372,7 +388,7 @@ describe(FilesManager.name, () => {const taskEmitSpy = jest.spyOn(FileTaskEvent, 'emit')const fileEmitSpy = jest.spyOn(FileEvent, 'emit')- await service.downloadFromUrl(user, space, 'https://example.org/file.txt')+ await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })expect(space.task.props.totalSize).toBe(55)expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DOWNLOAD, '/tmp/download.txt')diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex 7f7677b5..51fc10c2 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -1,7 +1,6 @@import { HttpService } from '@nestjs/axios'import { HttpStatus, Injectable, Logger } from '@nestjs/common'import archiver, { Archiver } from 'archiver'-import { AxiosResponse } from 'axios'import fs from 'node:fs'import path from 'node:path'import { Readable } from 'node:stream'@@ -27,7 +26,7 @@ import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'import { FILE_OPERATION } from '../constants/operations'import { DOCUMENT_TYPE, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'-import { CompressFileDto } from '../dto/file-operations.dto'+import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'import { FileDBProps } from '../interfaces/file-db-props.interface'import { FileLock } from '../interfaces/file-lock.interface'import { FileLockProps } from '../interfaces/file-props.interface'@@ -56,7 +55,7 @@ import {} from '../utils/files'import { SendFile } from '../utils/send-file'import { extractZip } from '../utils/unzip-file'-import { regExpPrivateIP } from '../utils/url-file'+import { downloadFile } from '../utils/download-file'import { FilesLockManager } from './files-lock-manager.service'import { FilesQueries } from './files-queries.service'import { FileEvent, FileTaskEvent } from '../events/file-events'@@ -503,54 +502,21 @@ export class FilesManager {await this.filesQueries.deleteFiles(space.dbFile, isDir, forceDeleteInDB)}- async downloadFromUrl(user: UserModel, space: SpaceEnv, url: string): Promise<void> {- this.logger.log({ tag: this.downloadFromUrl.name, msg: `${url}` })- // create lock+ async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {+ this.logger.log({ tag: this.downloadFromUrl.name, msg: `${downloadDto.url}` })const rPath = await uniqueFilePathFromDir(space.realPath)const dbFile = space.dbFiledbFile.path = path.join(dirName(dbFile.path), fileName(space.realPath))++ // create lockconst [ok, fileLock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.RESOURCE)if (!ok) {throw new LockConflict(fileLock, 'Conflicting lock')}- // tasking- if (space.task.cacheKey) {- let headRes: AxiosResponse-- try {- headRes = await this.http.axiosRef({ method: HTTP_METHOD.HEAD, url: url, maxRedirects: 1 })- } catch (e) {- // release lock- await this.filesLockManager.removeLock(fileLock.key)- this.logger.error({ tag: this.downloadFromUrl.name, msg: `${url} : ${e}` })- throw new FileError(HttpStatus.BAD_REQUEST, 'Unable to download file')- }- if (regExpPrivateIP.test(headRes.request.socket.remoteAddress)) {- // release lock- await this.filesLockManager.removeLock(fileLock.key)- // prevent SSRF attack- throw new FileError(HttpStatus.FORBIDDEN, 'Access to internal IP addresses is forbidden')- }-- // attempt to retrieve the Content-Length header- try {- if ('content-length' in headRes.headers) {- space.task.props.totalSize = Number(headRes.headers['content-length']) || null- }- } catch (e) {- this.logger.debug({ tag: this.downloadFromUrl.name, msg: `content-length : ${e}` })- }- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, rPath)- }// dotry {- const getRes = await this.http.axiosRef({ method: HTTP_METHOD.GET, url: url, responseType: 'stream', maxRedirects: 1 })- if (regExpPrivateIP.test(getRes.request.socket.remoteAddress)) {- // Prevent SSRF attacks and perform a DNS-rebinding check if a HEAD request has already been made- throw new FileError(HttpStatus.FORBIDDEN, 'Access to internal IP addresses is forbidden')- }- await writeFromStream(rPath, getRes.data)+ await downloadFile(this.http, downloadDto, rPath, space)} finally {// release lockawait this.filesLockManager.removeLock(fileLock.key)diff --git a/backend/src/applications/files/services/files-methods.service.ts b/backend/src/applications/files/services/files-methods.service.tsindex f534a6b0..c4b4e512 100644--- a/backend/src/applications/files/services/files-methods.service.ts+++ b/backend/src/applications/files/services/files-methods.service.ts@@ -87,7 +87,7 @@ export class FilesMethods {async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {checkFileName(space.realPath)try {- return await this.filesManager.downloadFromUrl(user, space, downloadDto.url)+ return await this.filesManager.downloadFromUrl(user, space, downloadDto)} catch (e) {this.handleError(space, FILE_OPERATION.DOWNLOAD, e)}diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.tsnew file mode 100644index 00000000..8f66003b--- /dev/null+++ b/backend/src/applications/files/utils/download-file.ts@@ -0,0 +1,71 @@+import type { HttpService } from '@nestjs/axios'+import type { SpaceEnv } from '../../spaces/models/space-env.model'+import type { AxiosResponse } from 'axios'+import { HTTP_METHOD } from '../../applications.constants'+import { FileError } from '../models/file-error'+import { HttpStatus } from '@nestjs/common'+import { FileTaskEvent } from '../events/file-events'+import { FILE_OPERATION } from '../constants/operations'+import { writeFromStream } from './files'+import { DownloadFileDto } from '../dto/file-operations.dto'++const parts = [+ // IPv4 loopback (127.0.0.0/8)+ '127\\.(?:\\d{1,3}\\.){2}\\d{1,3}',+ // IPv4 link-local (169.254.0.0/16)+ '169\\.254\\.\\d{1,3}\\.\\d{1,3}',+ // IPv4 Carrier-grade NAT (100.64.0.0/10)+ '100\\.(?:6[4-9]|[7-9]\\d|1[01]\\d|12[0-7])\\.\\d{1,3}\\.\\d{1,3}',+ // IPv4 private (10.0.0.0/8)+ '10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}',+ // IPv4 private (192.168.0.0/16)+ '192\\.168\\.\\d{1,3}\\.\\d{1,3}',+ // IPv4 private (172.16.0.0/12)+ '172\\.(?:1[6-9]|2\\d|3[0-1])\\.\\d{1,3}\\.\\d{1,3}',+ // IPv4 & IPv6 loopback+ '::1',+ '::',+ '0.0.0.0',+ // IPv6 Unique Local Address (fc00::/7)+ 'f[cd][0-9a-f]{2}:[0-9a-f:]+',+ // IPv6 link-local (fe80::/10)+ 'fe[89ab][0-9a-f]{2}:[0-9a-f:]+'+]++const regExpPrivateIP = new RegExp(`^(?:${parts.join('|')})$`, 'i')+const errorRegexpPrivateIP = 'Access to internal IP addresses is forbidden'++export async function downloadFile(http: HttpService, downloadDto: DownloadFileDto, dstPath: string, space?: SpaceEnv) {+ // dto must be validated by the caller+ const headRes: AxiosResponse = await http.axiosRef({ method: HTTP_METHOD.HEAD, url: downloadDto.url, maxRedirects: 1 })+ if (regExpPrivateIP.test(headRes.request.socket.remoteAddress)) {+ // prevent SSRF attack+ throw new FileError(HttpStatus.FORBIDDEN, errorRegexpPrivateIP)+ }++ // attempt to retrieve the Content-Length header+ const contentLength = 'content-length' in headRes.headers ? Number(headRes.headers['content-length']) || null : null+ if (!contentLength) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'Missing "content-length" header')+ }++ if (space) {+ if (space.willExceedQuota(contentLength)) {+ throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, 'Storage quota will be exceeded')+ }+ // tasking+ if (space.task.cacheKey) {+ space.task.props.totalSize = contentLength+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)+ }+ }++ const getRes = await http.axiosRef({ method: HTTP_METHOD.GET, url: downloadDto.url, responseType: 'stream', maxRedirects: 1 })+ if (regExpPrivateIP.test(getRes.request.socket.remoteAddress)) {+ // close request+ getRes.data?.destroy()+ // Prevent SSRF attacks and perform a DNS-rebinding check if a HEAD request has already been made+ throw new FileError(HttpStatus.FORBIDDEN, errorRegexpPrivateIP)+ }+ await writeFromStream(dstPath, getRes.data)+}diff --git a/backend/src/applications/files/utils/url-file.ts b/backend/src/applications/files/utils/url-file.tsdeleted file mode 100644index f613a8f2..00000000--- a/backend/src/applications/files/utils/url-file.ts+++ /dev/null@@ -1,24 +0,0 @@-const parts = [- // IPv4 loopback (127.0.0.0/8)- '127\\.(?:\\d{1,3}\\.){2}\\d{1,3}',- // IPv4 link-local (169.254.0.0/16)- '169\\.254\\.\\d{1,3}\\.\\d{1,3}',- // IPv4 Carrier-grade NAT (100.64.0.0/10)- '100\\.(?:6[4-9]|[7-9]\\d|1[01]\\d|12[0-7])\\.\\d{1,3}\\.\\d{1,3}',- // IPv4 private (10.0.0.0/8)- '10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}',- // IPv4 private (192.168.0.0/16)- '192\\.168\\.\\d{1,3}\\.\\d{1,3}',- // IPv4 private (172.16.0.0/12)- '172\\.(?:1[6-9]|2\\d|3[0-1])\\.\\d{1,3}\\.\\d{1,3}',- // IPv4 & IPv6 loopback- '::1',- '::',- '0.0.0.0',- // IPv6 Unique Local Address (fc00::/7)- 'f[cd][0-9a-f]{2}:[0-9a-f:]+',- // IPv6 link-local (fe80::/10)- 'fe[89ab][0-9a-f]{2}:[0-9a-f:]+'-]--export const regExpPrivateIP = new RegExp(`^(?:${parts.join('|')})$`, 'i')