Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:sync): validate uploads before promoting temp files
johaven346f8cbd245f
diff --git a/backend/src/applications/files/interfaces/save-stream.interface.ts b/backend/src/applications/files/interfaces/save-stream.interface.tsnew file mode 100644index 00000000..d2deb6ba--- /dev/null+++ b/backend/src/applications/files/interfaces/save-stream.interface.ts@@ -0,0 +1,14 @@+import { LOCK_DEPTH } from '../../webdav/constants/webdav'++export interface SaveStreamTmpFileValidationContext {+ tmpPath: string+ realPath: string+ checksum?: string+}++export interface SaveStreamOptions {+ dav?: { depth: LOCK_DEPTH; lockTokens: string[] }+ checksumAlg?: string+ tmpPath?: string+ validateTmpFile?: (ctx: SaveStreamTmpFileValidationContext) => Promise<void>+}diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.tsindex 13b42a6f..220d6960 100644--- a/backend/src/applications/files/services/files-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-manager.service.spec.ts@@ -264,6 +264,29 @@ describe(FilesManager.name, () => {expect(filesLockManager.create).not.toHaveBeenCalled()expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalled()})++ it('should validate tmp stream before moving it to the destination', async () => {+ const space = makeSpace()+ const tmpPath = '/data/users/john/tmp/sync-in-file.txt'+ const validationError = new FileError(HttpStatus.BAD_REQUEST, 'Invalid sync upload')+ const validateTmpFile = vi.fn().mockRejectedValue(validationError)+ const emitSpy = vi.spyOn(FileEvent, 'emit')+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true, [tmpPath]: true }, false)++ await expect(+ service.saveStream(user, space, { method: 'PUT', headers: {}, raw: Readable.from(['chunk']) } as any, {+ tmpPath,+ checksumAlg: 'sha256',+ validateTmpFile+ })+ ).rejects.toEqual(validationError)++ expect(validateTmpFile).toHaveBeenCalledWith({ tmpPath, realPath: space.realPath, checksum: 'sha256-abc' })+ expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalledWith(tmpPath, expect.anything(), 0, 'sha256')+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(tmpPath)+ expect(emitSpy).not.toHaveBeenCalled()+ })})describe('saveMultipart', () => {diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.tsindex fa66f9f0..3c0db399 100644--- a/backend/src/applications/files/services/files-manager.service.ts+++ b/backend/src/applications/files/services/files-manager.service.ts@@ -26,6 +26,7 @@ import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'import { FileDBProps } from '../interfaces/file-db-props.interface'import { FileLock } from '../interfaces/file-lock.interface'import { FileLockProps } from '../interfaces/file-props.interface'+import { SaveStreamOptions } from '../interfaces/save-stream.interface'import { FileError, SourceCleanupError } from '../models/file-error'import { LockConflict } from '../models/file-lock-error'import {@@ -90,18 +91,10 @@ export class FilesManager {user: UserModel,space: SpaceEnv,req: FastifyAuthenticatedRequest,- options: {- checksumAlg: string- tmpPath?: string- }+ options: SaveStreamOptions & { checksumAlg: string }): Promise<string>async saveStream(user: UserModel, space: SpaceEnv, req: FastifyAuthenticatedRequest, options?: any): Promise<boolean>- async saveStream(- user: UserModel,- space: SpaceEnv,- req: FastifyAuthenticatedRequest,- options?: { dav?: { depth: LOCK_DEPTH; lockTokens: string[] }; checksumAlg?: string; tmpPath?: string }- ): Promise<boolean | string> {+ async saveStream(user: UserModel, space: SpaceEnv, req: FastifyAuthenticatedRequest, options?: SaveStreamOptions): Promise<boolean | string> {// If tmpPath is used, we lock the final destination during the transfer// space.realPath is replaced by tmpPath (if allowed). If the move operation failed, we remove the tmp filethis.checkNotTrashRepository(space)@@ -136,6 +129,7 @@ export class FilesManager {fileLock = lock}const fileEventAction = fExists ? ACTION.UPDATE : ACTION.ADD+ let fileWritten = falsetry {// Check rangelet startRange = 0@@ -162,25 +156,31 @@ export class FilesManager {await writeFromStream(options?.tmpPath || space.realPath, req.raw, startRange)}if (options?.tmpPath) {+ await options.validateTmpFile?.({ tmpPath: options.tmpPath, realPath: space.realPath, checksum })try {// ensure parent path existsawait makeDir(path.dirname(space.realPath), true)// move the uploaded file to destinationawait moveFiles(options.tmpPath, space.realPath, true)+ fileWritten = true} catch (e) {// cleanup tmp fileawait removeFiles(options.tmpPath)this.logger.error({ tag: this.saveStream.name, msg: `unable to move ${options.tmpPath} -> ${space.realPath} : ${e}` })throw new FileError(HttpStatus.INTERNAL_SERVER_ERROR, 'Unable to move tmp file to dst file')}+ } else {+ fileWritten = true}if (options?.checksumAlg) {return checksum}return fExists} finally {- // emit file event- FileEvent.emit('event', { user, space, action: fileEventAction, rPath: space.realPath })+ if (fileWritten) {+ // emit file event+ FileEvent.emit('event', { user, space, action: fileEventAction, rPath: space.realPath })+ }if (fileLock) {try {await this.filesLockManager.removeLock(fileLock.key)diff --git a/backend/src/applications/sync/services/sync-manager.service.spec.ts b/backend/src/applications/sync/services/sync-manager.service.spec.tsindex 369817f8..ac0ad836 100644--- a/backend/src/applications/sync/services/sync-manager.service.spec.ts+++ b/backend/src/applications/sync/services/sync-manager.service.spec.ts@@ -163,20 +163,27 @@ describe(SyncManager.name, () => {vi.mocked(touchFile).mockResolvedValue(undefined)const r = await service.upload(req as any, dto as any)- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, { tmpPath: expect.any(String), checksumAlg: SYNC_CHECKSUM_ALG })+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {+ tmpPath: expect.any(String),+ checksumAlg: SYNC_CHECKSUM_ALG,+ validateTmpFile: expect.any(Function)+ })expect(touchFile).toHaveBeenCalledWith('/tmp/up.bin', 1710000000)expect(r).toEqual({ ino: 123 })expect(removeFiles).not.toHaveBeenCalled()})- it('should reject when checksum mismatches and remove tmp', async () => {+ it('should reject when checksum mismatches and preserve tmp', async () => {const req = makeReq()const dto = { checksum: 'abc', size: 10, mtime: 1710000000 }- filesManager.saveStream.mockResolvedValue('bad')+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath, checksum: 'bad' })+ })fsPromises.stat.mockResolvedValue({ size: 10, ino: 123, mtime: new Date(1710000000 * 1000) })await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)- expect(removeFiles).toHaveBeenCalledWith(expect.any(String))+ expect(removeFiles).not.toHaveBeenCalled()+ expect(touchFile).not.toHaveBeenCalled()})it('should upload without checksum', async () => {@@ -186,19 +193,41 @@ describe(SyncManager.name, () => {fsPromises.stat.mockResolvedValue({ size: 5, ino: 321, mtime: new Date(1710000100 * 1000) })const r = await service.upload(req as any, dto as any)- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, { tmpPath: expect.any(String) })+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {+ tmpPath: expect.any(String),+ validateTmpFile: expect.any(Function)+ })expect(touchFile).toHaveBeenCalledWith('/tmp/up2.bin', 1710000100)expect(r).toEqual({ ino: 321 })})- it('should reject when size mismatches and remove tmp', async () => {+ it('should reject when size mismatches and preserve tmp', async () => {const req = makeReq()const dto = { size: 10, mtime: 1710000100 }- filesManager.saveStream.mockResolvedValue(undefined)+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath })+ })fsPromises.stat.mockResolvedValue({ size: 99, ino: 321, mtime: new Date(1710000100 * 1000) })await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)- expect(removeFiles).toHaveBeenCalledWith(expect.any(String))+ expect(removeFiles).not.toHaveBeenCalled()+ expect(touchFile).not.toHaveBeenCalled()+ })++ it('should reject when tmp file exceeds quota and preserve tmp', async () => {+ const req = makeReq({+ space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota: vi.fn().mockReturnValue(true) }+ })+ const dto = { size: 10, mtime: 1710000100 }+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath })+ })+ fsPromises.stat.mockResolvedValue({ size: 10, ino: 321, mtime: new Date(1710000100 * 1000) })++ await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)+ expect(req.space.willExceedQuota).toHaveBeenCalledWith(10)+ expect(removeFiles).not.toHaveBeenCalled()+ expect(touchFile).not.toHaveBeenCalled()})})diff --git a/backend/src/applications/sync/services/sync-manager.service.ts b/backend/src/applications/sync/services/sync-manager.service.tsindex c98240b7..5f63c8ec 100644--- a/backend/src/applications/sync/services/sync-manager.service.ts+++ b/backend/src/applications/sync/services/sync-manager.service.ts@@ -8,7 +8,7 @@ import { FILE_OPERATION } from '../../files/constants/operations'import { FileError } from '../../files/models/file-error'import { LockConflict } from '../../files/models/file-lock-error'import { FilesManager } from '../../files/services/files-manager.service'-import { checksumFile, isPathExists, isPathIsDir, removeFiles, touchFile } from '../../files/utils/files'+import { checksumFile, isPathExists, isPathIsDir, touchFile } from '../../files/utils/files'import { SendFile } from '../../files/utils/send-file'import { ParseDiffContext } from '../../spaces/interfaces/space-diff.interface'import { FastifySpaceRequest } from '../../spaces/interfaces/space-request.interface'@@ -20,7 +20,7 @@ import { SyncCopyMoveDto, SyncDiffDto, SyncMakeDto, SyncPropsDto } from '../dtosimport { SyncUploadDto } from '../dtos/sync-upload.dto'import { SyncFileSpecialStats, SyncFileStats } from '../interfaces/sync-diff.interface'import { SyncPathSettings } from '../interfaces/sync-path.interface'-import { getTmpFilePath } from '../utils/functions'+import { getSyncTmpFilePath } from '../utils/functions'import { SYNC_PATH_TO_SPACE_SEGMENTS } from '../utils/routes'import { SyncQueries } from './sync-queries.service'@@ -45,29 +45,28 @@ export class SyncManager {}async upload(req: FastifySpaceRequest, syncUploadDto: SyncUploadDto): Promise<{ ino: number }> {- const tmpPath = getTmpFilePath(req.space.realPath)+ const tmpPath = getSyncTmpFilePath(req.space.realPath)try {- if (syncUploadDto.checksum) {- const checksum = await this.filesManager.saveStream(req.user, req.space, req, { tmpPath: tmpPath, checksumAlg: SYNC_CHECKSUM_ALG })- if (checksum !== syncUploadDto.checksum) {- await removeFiles(tmpPath)- this.handleError(req.space, req.method, new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical'))+ await this.filesManager.saveStream(req.user, req.space, req, {+ tmpPath: tmpPath,+ ...(syncUploadDto.checksum && { checksumAlg: SYNC_CHECKSUM_ALG }),+ validateTmpFile: async ({ tmpPath, checksum }) => {+ const tmpStats = await fs.stat(tmpPath)+ if (tmpStats.size !== syncUploadDto.size) {+ throw new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${tmpStats.size} != ${syncUploadDto.size}`)+ }+ if (req.space.storageQuota && req.space.willExceedQuota(tmpStats.size)) {+ throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, 'Storage quota exceeded')+ }+ if (syncUploadDto.checksum && checksum !== syncUploadDto.checksum) {+ throw new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical')+ }}- } else {- await this.filesManager.saveStream(req.user, req.space, req, { tmpPath: tmpPath })- }- const fileStats = await fs.stat(req.space.realPath)- if (fileStats.size !== syncUploadDto.size) {- await removeFiles(tmpPath)- this.handleError(- req.space,- req.method,- new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${fileStats.size} != ${syncUploadDto.size}`)- )- }+ })// update mtimeawait touchFile(req.space.realPath, syncUploadDto.mtime)// return inode number+ const fileStats = await fs.stat(req.space.realPath)return { ino: fileStats.ino }} catch (e) {this.handleError(req.space, req.method, e)diff --git a/backend/src/applications/sync/utils/functions.ts b/backend/src/applications/sync/utils/functions.tsindex 9e8c4a51..ab78413e 100644--- a/backend/src/applications/sync/utils/functions.ts+++ b/backend/src/applications/sync/utils/functions.ts@@ -6,7 +6,7 @@ import { SYNC_FILE_NAME_PREFIX } from '../constants/sync'const MAX_PATH_FILTER_LENGTH = 200const MAX_PATH_FILTER_REPETITIONS = 25-export function getTmpFilePath(rPath: string): string {+export function getSyncTmpFilePath(rPath: string): string {return `${path.dirname(rPath)}/${SYNC_FILE_NAME_PREFIX}${path.basename(rPath)}`}