Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:sync): validate uploads before promoting temp files

johaven346f8cbd245f
diff --git a/backend/src/applications/files/interfaces/save-stream.interface.ts b/backend/src/applications/files/interfaces/save-stream.interface.ts
new file mode 100644
index 00000000..d2deb6ba
--- /dev/null
+++ b/backend/src/applications/files/interfaces/save-stream.interface.ts
@@ -0,0 +1,14 @@
+import { LOCK_DEPTH } from '../../webdav/constants/webdav'
+
+export interface SaveStreamTmpFileValidationContext {
+ tmpPath: string
+ realPath: string
+ checksum?: string
+}
+
+export interface SaveStreamOptions {
+ dav?: { depth: LOCK_DEPTH; lockTokens: string[] }
+ checksumAlg?: string
+ tmpPath?: string
+ validateTmpFile?: (ctx: SaveStreamTmpFileValidationContext) => Promise<void>
+}
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index 13b42a6f..220d6960 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -264,6 +264,29 @@ describe(FilesManager.name, () => {
expect(filesLockManager.create).not.toHaveBeenCalled()
expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalled()
})
+
+ it('should validate tmp stream before moving it to the destination', async () => {
+ const space = makeSpace()
+ const tmpPath = '/data/users/john/tmp/sync-in-file.txt'
+ const validationError = new FileError(HttpStatus.BAD_REQUEST, 'Invalid sync upload')
+ const validateTmpFile = vi.fn().mockRejectedValue(validationError)
+ const emitSpy = vi.spyOn(FileEvent, 'emit')
+ setPathExists({ [space.realPath]: false, [path.dirname(space.realPath)]: true, [tmpPath]: true }, false)
+
+ await expect(
+ service.saveStream(user, space, { method: 'PUT', headers: {}, raw: Readable.from(['chunk']) } as any, {
+ tmpPath,
+ checksumAlg: 'sha256',
+ validateTmpFile
+ })
+ ).rejects.toEqual(validationError)
+
+ expect(validateTmpFile).toHaveBeenCalledWith({ tmpPath, realPath: space.realPath, checksum: 'sha256-abc' })
+ expect(filesUtils.writeFromStreamAndChecksum).toHaveBeenCalledWith(tmpPath, expect.anything(), 0, 'sha256')
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ expect(filesUtils.removeFiles).not.toHaveBeenCalledWith(tmpPath)
+ expect(emitSpy).not.toHaveBeenCalled()
+ })
})
describe('saveMultipart', () => {
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index fa66f9f0..3c0db399 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -26,6 +26,7 @@ import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
import { FileDBProps } from '../interfaces/file-db-props.interface'
import { FileLock } from '../interfaces/file-lock.interface'
import { FileLockProps } from '../interfaces/file-props.interface'
+import { SaveStreamOptions } from '../interfaces/save-stream.interface'
import { FileError, SourceCleanupError } from '../models/file-error'
import { LockConflict } from '../models/file-lock-error'
import {
@@ -90,18 +91,10 @@ export class FilesManager {
user: UserModel,
space: SpaceEnv,
req: FastifyAuthenticatedRequest,
- options: {
- checksumAlg: string
- tmpPath?: string
- }
+ options: SaveStreamOptions & { checksumAlg: string }
): Promise<string>
async saveStream(user: UserModel, space: SpaceEnv, req: FastifyAuthenticatedRequest, options?: any): Promise<boolean>
- async saveStream(
- user: UserModel,
- space: SpaceEnv,
- req: FastifyAuthenticatedRequest,
- options?: { dav?: { depth: LOCK_DEPTH; lockTokens: string[] }; checksumAlg?: string; tmpPath?: string }
- ): Promise<boolean | string> {
+ async saveStream(user: UserModel, space: SpaceEnv, req: FastifyAuthenticatedRequest, options?: SaveStreamOptions): Promise<boolean | string> {
// If tmpPath is used, we lock the final destination during the transfer
// space.realPath is replaced by tmpPath (if allowed). If the move operation failed, we remove the tmp file
this.checkNotTrashRepository(space)
@@ -136,6 +129,7 @@ export class FilesManager {
fileLock = lock
}
const fileEventAction = fExists ? ACTION.UPDATE : ACTION.ADD
+ let fileWritten = false
try {
// Check range
let startRange = 0
@@ -162,25 +156,31 @@ export class FilesManager {
await writeFromStream(options?.tmpPath || space.realPath, req.raw, startRange)
}
if (options?.tmpPath) {
+ await options.validateTmpFile?.({ tmpPath: options.tmpPath, realPath: space.realPath, checksum })
try {
// ensure parent path exists
await makeDir(path.dirname(space.realPath), true)
// move the uploaded file to destination
await moveFiles(options.tmpPath, space.realPath, true)
+ fileWritten = true
} catch (e) {
// cleanup tmp file
await removeFiles(options.tmpPath)
this.logger.error({ tag: this.saveStream.name, msg: `unable to move ${options.tmpPath} -> ${space.realPath} : ${e}` })
throw new FileError(HttpStatus.INTERNAL_SERVER_ERROR, 'Unable to move tmp file to dst file')
}
+ } else {
+ fileWritten = true
}
if (options?.checksumAlg) {
return checksum
}
return fExists
} finally {
- // emit file event
- FileEvent.emit('event', { user, space, action: fileEventAction, rPath: space.realPath })
+ if (fileWritten) {
+ // emit file event
+ FileEvent.emit('event', { user, space, action: fileEventAction, rPath: space.realPath })
+ }
if (fileLock) {
try {
await this.filesLockManager.removeLock(fileLock.key)
diff --git a/backend/src/applications/sync/services/sync-manager.service.spec.ts b/backend/src/applications/sync/services/sync-manager.service.spec.ts
index 369817f8..ac0ad836 100644
--- a/backend/src/applications/sync/services/sync-manager.service.spec.ts
+++ b/backend/src/applications/sync/services/sync-manager.service.spec.ts
@@ -163,20 +163,27 @@ describe(SyncManager.name, () => {
vi.mocked(touchFile).mockResolvedValue(undefined)
const r = await service.upload(req as any, dto as any)
- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, { tmpPath: expect.any(String), checksumAlg: SYNC_CHECKSUM_ALG })
+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {
+ tmpPath: expect.any(String),
+ checksumAlg: SYNC_CHECKSUM_ALG,
+ validateTmpFile: expect.any(Function)
+ })
expect(touchFile).toHaveBeenCalledWith('/tmp/up.bin', 1710000000)
expect(r).toEqual({ ino: 123 })
expect(removeFiles).not.toHaveBeenCalled()
})
- it('should reject when checksum mismatches and remove tmp', async () => {
+ it('should reject when checksum mismatches and preserve tmp', async () => {
const req = makeReq()
const dto = { checksum: 'abc', size: 10, mtime: 1710000000 }
- filesManager.saveStream.mockResolvedValue('bad')
+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {
+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath, checksum: 'bad' })
+ })
fsPromises.stat.mockResolvedValue({ size: 10, ino: 123, mtime: new Date(1710000000 * 1000) })
await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)
- expect(removeFiles).toHaveBeenCalledWith(expect.any(String))
+ expect(removeFiles).not.toHaveBeenCalled()
+ expect(touchFile).not.toHaveBeenCalled()
})
it('should upload without checksum', async () => {
@@ -186,19 +193,41 @@ describe(SyncManager.name, () => {
fsPromises.stat.mockResolvedValue({ size: 5, ino: 321, mtime: new Date(1710000100 * 1000) })
const r = await service.upload(req as any, dto as any)
- expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, { tmpPath: expect.any(String) })
+ expect(filesManager.saveStream).toHaveBeenCalledWith(req.user, req.space, req, {
+ tmpPath: expect.any(String),
+ validateTmpFile: expect.any(Function)
+ })
expect(touchFile).toHaveBeenCalledWith('/tmp/up2.bin', 1710000100)
expect(r).toEqual({ ino: 321 })
})
- it('should reject when size mismatches and remove tmp', async () => {
+ it('should reject when size mismatches and preserve tmp', async () => {
const req = makeReq()
const dto = { size: 10, mtime: 1710000100 }
- filesManager.saveStream.mockResolvedValue(undefined)
+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {
+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath })
+ })
fsPromises.stat.mockResolvedValue({ size: 99, ino: 321, mtime: new Date(1710000100 * 1000) })
await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)
- expect(removeFiles).toHaveBeenCalledWith(expect.any(String))
+ expect(removeFiles).not.toHaveBeenCalled()
+ expect(touchFile).not.toHaveBeenCalled()
+ })
+
+ it('should reject when tmp file exceeds quota and preserve tmp', async () => {
+ const req = makeReq({
+ space: { realPath: '/tmp/up.bin', url: '/space/up.bin', storageQuota: 100, willExceedQuota: vi.fn().mockReturnValue(true) }
+ })
+ const dto = { size: 10, mtime: 1710000100 }
+ filesManager.saveStream.mockImplementation(async (_user, _space, _req, options) => {
+ await options.validateTmpFile({ tmpPath: '/tmp/sync-in-file', realPath: req.space.realPath })
+ })
+ fsPromises.stat.mockResolvedValue({ size: 10, ino: 321, mtime: new Date(1710000100 * 1000) })
+
+ await expect(service.upload(req as any, dto as any)).rejects.toBeInstanceOf(HttpException)
+ expect(req.space.willExceedQuota).toHaveBeenCalledWith(10)
+ expect(removeFiles).not.toHaveBeenCalled()
+ expect(touchFile).not.toHaveBeenCalled()
})
})
diff --git a/backend/src/applications/sync/services/sync-manager.service.ts b/backend/src/applications/sync/services/sync-manager.service.ts
index c98240b7..5f63c8ec 100644
--- a/backend/src/applications/sync/services/sync-manager.service.ts
+++ b/backend/src/applications/sync/services/sync-manager.service.ts
@@ -8,7 +8,7 @@ import { FILE_OPERATION } from '../../files/constants/operations'
import { FileError } from '../../files/models/file-error'
import { LockConflict } from '../../files/models/file-lock-error'
import { FilesManager } from '../../files/services/files-manager.service'
-import { checksumFile, isPathExists, isPathIsDir, removeFiles, touchFile } from '../../files/utils/files'
+import { checksumFile, isPathExists, isPathIsDir, touchFile } from '../../files/utils/files'
import { SendFile } from '../../files/utils/send-file'
import { ParseDiffContext } from '../../spaces/interfaces/space-diff.interface'
import { FastifySpaceRequest } from '../../spaces/interfaces/space-request.interface'
@@ -20,7 +20,7 @@ import { SyncCopyMoveDto, SyncDiffDto, SyncMakeDto, SyncPropsDto } from '../dtos
import { SyncUploadDto } from '../dtos/sync-upload.dto'
import { SyncFileSpecialStats, SyncFileStats } from '../interfaces/sync-diff.interface'
import { SyncPathSettings } from '../interfaces/sync-path.interface'
-import { getTmpFilePath } from '../utils/functions'
+import { getSyncTmpFilePath } from '../utils/functions'
import { SYNC_PATH_TO_SPACE_SEGMENTS } from '../utils/routes'
import { SyncQueries } from './sync-queries.service'
@@ -45,29 +45,28 @@ export class SyncManager {
}
async upload(req: FastifySpaceRequest, syncUploadDto: SyncUploadDto): Promise<{ ino: number }> {
- const tmpPath = getTmpFilePath(req.space.realPath)
+ const tmpPath = getSyncTmpFilePath(req.space.realPath)
try {
- if (syncUploadDto.checksum) {
- const checksum = await this.filesManager.saveStream(req.user, req.space, req, { tmpPath: tmpPath, checksumAlg: SYNC_CHECKSUM_ALG })
- if (checksum !== syncUploadDto.checksum) {
- await removeFiles(tmpPath)
- this.handleError(req.space, req.method, new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical'))
+ await this.filesManager.saveStream(req.user, req.space, req, {
+ tmpPath: tmpPath,
+ ...(syncUploadDto.checksum && { checksumAlg: SYNC_CHECKSUM_ALG }),
+ validateTmpFile: async ({ tmpPath, checksum }) => {
+ const tmpStats = await fs.stat(tmpPath)
+ if (tmpStats.size !== syncUploadDto.size) {
+ throw new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${tmpStats.size} != ${syncUploadDto.size}`)
+ }
+ if (req.space.storageQuota && req.space.willExceedQuota(tmpStats.size)) {
+ throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, 'Storage quota exceeded')
+ }
+ if (syncUploadDto.checksum && checksum !== syncUploadDto.checksum) {
+ throw new FileError(HttpStatus.BAD_REQUEST, 'checksums are not identical')
+ }
}
- } else {
- await this.filesManager.saveStream(req.user, req.space, req, { tmpPath: tmpPath })
- }
- const fileStats = await fs.stat(req.space.realPath)
- if (fileStats.size !== syncUploadDto.size) {
- await removeFiles(tmpPath)
- this.handleError(
- req.space,
- req.method,
- new FileError(HttpStatus.BAD_REQUEST, `sizes are not identical : ${fileStats.size} != ${syncUploadDto.size}`)
- )
- }
+ })
// update mtime
await touchFile(req.space.realPath, syncUploadDto.mtime)
// return inode number
+ const fileStats = await fs.stat(req.space.realPath)
return { ino: fileStats.ino }
} catch (e) {
this.handleError(req.space, req.method, e)
diff --git a/backend/src/applications/sync/utils/functions.ts b/backend/src/applications/sync/utils/functions.ts
index 9e8c4a51..ab78413e 100644
--- a/backend/src/applications/sync/utils/functions.ts
+++ b/backend/src/applications/sync/utils/functions.ts
@@ -6,7 +6,7 @@ import { SYNC_FILE_NAME_PREFIX } from '../constants/sync'
const MAX_PATH_FILTER_LENGTH = 200
const MAX_PATH_FILTER_REPETITIONS = 25
-export function getTmpFilePath(rPath: string): string {
+export function getSyncTmpFilePath(rPath: string): string {
return `${path.dirname(rPath)}/${SYNC_FILE_NAME_PREFIX}${path.basename(rPath)}`
}