Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): stage downloads and archives in user tmp paths before publishing

johaven1363899d1a90
diff --git a/backend/src/applications/files/interfaces/download-file.interface.ts b/backend/src/applications/files/interfaces/download-file.interface.ts
index e9ef3204..66450bf5 100644
--- a/backend/src/applications/files/interfaces/download-file.interface.ts
+++ b/backend/src/applications/files/interfaces/download-file.interface.ts
@@ -9,6 +9,7 @@ export interface DownloadFileContentInfo {
export interface DownloadFileOptions {
allowPrivateIP?: boolean
space?: SpaceEnv
+ publishedPath?: string
getContentInfo?: boolean
maxSize?: number
}
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index c5c48e22..858dd74c 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -95,12 +95,17 @@ describe(FilesManager.name, () => {
directory: jest.Mock
file: jest.Mock
finalize: jest.Mock
+ abort: jest.Mock
}
archive.directory = jest.fn().mockReturnValue(archive)
archive.file = jest.fn().mockReturnValue(archive)
archive.finalize = jest.fn().mockImplementation(async () => {
archive.end()
})
+ archive.abort = jest.fn().mockImplementation(() => {
+ archive.end()
+ return archive
+ })
;(archiver as unknown as jest.Mock).mockReturnValueOnce(archive as any)
jest.spyOn(fs, 'createWriteStream').mockReturnValue(new PassThrough() as any)
return archive
@@ -165,6 +170,7 @@ describe(FilesManager.name, () => {
jest.spyOn(filesUtils, 'isPathIsDir').mockResolvedValue(false)
jest.spyOn(filesUtils, 'makeDir').mockResolvedValue('/tmp' as any)
jest.spyOn(filesUtils, 'makeTempDir').mockResolvedValue('/tmp/extract')
+ jest.spyOn(filesUtils, 'tempFilePath').mockReturnValue('/tmp/staged-file')
jest.spyOn(filesUtils, 'writeFromStream').mockResolvedValue(undefined)
jest.spyOn(filesUtils, 'writeFromStreamAndChecksum').mockResolvedValue('sha256-abc')
jest.spyOn(filesUtils, 'moveFiles').mockResolvedValue(undefined)
@@ -880,6 +886,7 @@ describe(FilesManager.name, () => {
it('should handle HEAD+GET and emit task watch/event', async () => {
const space = makeSpace({ task: { cacheKey: 'task-1', props: {} } })
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')
http.axiosRef
.mockResolvedValueOnce({
headers: { 'content-length': '55' },
@@ -895,8 +902,21 @@ describe(FilesManager.name, () => {
await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })
expect(space.task.props.totalSize).toBe(55)
- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.DOWNLOAD, '/tmp/download.txt')
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/tmp/download.txt', expect.anything(), 0, 55)
+ expect(taskEmitSpy).toHaveBeenCalledWith(
+ 'startWatch',
+ space,
+ FILE_OPERATION.DOWNLOAD,
+ '/tmp/download.txt',
+ '/data/users/john/tmp/download.txt-download-uuid'
+ )
+ expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', expect.anything(), 0, 55)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid', '/tmp/download.txt')
+ expect(filesLockManager.create).toHaveBeenCalledWith(
+ user,
+ expect.objectContaining({ path: 'download.txt' }),
+ expect.any(String),
+ DEPTH.RESOURCE
+ )
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
expect(fileEmitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })
})
@@ -905,6 +925,7 @@ describe(FilesManager.name, () => {
const error = new FileError(HttpStatus.PAYLOAD_TOO_LARGE, FILE_ERROR_MESSAGES.MAX_FILE_SIZE_EXCEEDED)
const space = makeSpace()
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')
;(filesUtils.writeFromStream as jest.Mock).mockRejectedValueOnce(error)
http.axiosRef
.mockResolvedValueOnce({
@@ -919,7 +940,32 @@ describe(FilesManager.name, () => {
await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBe(error)
- expect(filesUtils.removeFiles).toHaveBeenCalledWith('/tmp/download.txt')
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid')
+ expect(filesUtils.moveFiles).not.toHaveBeenCalled()
+ expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
+ expect(fileEmitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })
+ })
+
+ it('should cleanup temporary file and skip ADD event when publishing download fails', async () => {
+ const error = new Error('move failed')
+ const space = makeSpace()
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/download.txt')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/download.txt-download-uuid')
+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)
+ http.axiosRef
+ .mockResolvedValueOnce({
+ headers: { 'content-length': '55' },
+ request: { socket: { remoteAddress: '8.8.8.8' } }
+ })
+ .mockResolvedValueOnce({
+ data: Readable.from(['abc']),
+ request: { socket: { remoteAddress: '8.8.8.8' } }
+ })
+ const fileEmitSpy = jest.spyOn(FileEvent, 'emit')
+
+ await expect(service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })).rejects.toBe(error)
+
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/download.txt-download-uuid')
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
expect(fileEmitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/download.txt' })
})
@@ -929,6 +975,7 @@ describe(FilesManager.name, () => {
it('should archive files and emit events', async () => {
const archive = createArchiveMock()
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
;(filesUtils.isPathIsDir as jest.Mock).mockImplementation(async (p: string) => p.endsWith('/dir'))
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
const dto = {
@@ -948,12 +995,21 @@ describe(FilesManager.name, () => {
expect(archive.directory).toHaveBeenCalled()
expect(archive.file).toHaveBeenCalled()
expect(archive.finalize).toHaveBeenCalled()
- expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, FILE_OPERATION.COMPRESS, '/tmp/archive.tar.gz')
+ expect(taskEmitSpy).toHaveBeenCalledWith(
+ 'startWatch',
+ space,
+ FILE_OPERATION.COMPRESS,
+ '/tmp/archive.tar.gz',
+ '/data/users/john/tmp/archive.tar.gz-compress-uuid'
+ )
+ expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', { highWaterMark: expect.any(Number) })
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid', '/tmp/archive.tar.gz')
})
it('should allow archive export from trash when compressInDirectory is false', async () => {
const archive = createArchiveMock()
;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive-trash.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid')
;(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)
const emitSpy = jest.spyOn(FileEvent, 'emit')
const space = makeTrashSpace({
@@ -971,8 +1027,121 @@ describe(FilesManager.name, () => {
await expect(service.compress(user, space, dto)).resolves.toBeUndefined()
expect(filesLockManager.create).not.toHaveBeenCalled()
expect(archive.file).toHaveBeenCalled()
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-trash.tar.gz-compress-uuid', '/tmp/archive-trash.tar.gz')
expect(emitSpy).toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive-trash.tar.gz' })
})
+
+ it('should cleanup temporary archive and skip ADD event when publishing archive fails', async () => {
+ const archive = createArchiveMock()
+ const error = new Error('move failed')
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ ;(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)
+ ;(filesUtils.moveFiles as jest.Mock).mockRejectedValueOnce(error)
+ const emitSpy = jest.spyOn(FileEvent, 'emit')
+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
+ const dto = {
+ name: 'archive',
+ extension: 'tar.gz',
+ compressInDirectory: false,
+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
+ } as any
+
+ await expect(service.compress(user, space, dto)).rejects.toBe(error)
+
+ expect(archive.abort).toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan((filesUtils.removeFiles as jest.Mock).mock.invocationCallOrder[0])
+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
+ })
+
+ it('should abort archive pipeline and cleanup temporary file when preparing an entry fails', async () => {
+ const archive = createArchiveMock()
+ const error = new Error('unable to prepare entry')
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ ;(filesUtils.isPathIsDir as jest.Mock).mockRejectedValueOnce(error)
+ const emitSpy = jest.spyOn(FileEvent, 'emit')
+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
+ const dto = {
+ name: 'archive',
+ extension: 'tar.gz',
+ compressInDirectory: false,
+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
+ } as any
+
+ await expect(service.compress(user, space, dto)).rejects.toBe(error)
+
+ expect(archive.abort).toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ expect(archive.abort.mock.invocationCallOrder[0]).toBeLessThan((filesUtils.removeFiles as jest.Mock).mock.invocationCallOrder[0])
+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
+ })
+
+ it('should cleanup temporary archive without waiting for finalize after pipeline failure', async () => {
+ const archive = createArchiveMock()
+ const error = new Error('pipeline failed')
+ archive.finalize.mockImplementationOnce(() => {
+ archive.emit('error', error)
+ return new Promise<void>(() => undefined)
+ })
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ ;(filesUtils.isPathIsDir as jest.Mock).mockResolvedValueOnce(false)
+ const emitSpy = jest.spyOn(FileEvent, 'emit')
+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
+ const dto = {
+ name: 'archive',
+ extension: 'tar.gz',
+ compressInDirectory: false,
+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
+ } as any
+
+ await expect(service.compress(user, space, dto)).rejects.toBe(error)
+
+ expect(archive.abort).toHaveBeenCalled()
+ expect(archive.destroyed).toBe(true)
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ expect(emitSpy).not.toHaveBeenCalledWith('event', { user, space, action: ACTION.ADD, rPath: '/tmp/archive.tar.gz' })
+ })
+
+ it('should not append entries after archive pipeline cancellation', async () => {
+ const archive = createArchiveMock()
+ const error = new Error('pipeline failed')
+ let resolveEntry!: (isDir: boolean) => void
+ let entryStarted!: () => void
+ const entryStartedPromise = new Promise<void>((resolve) => {
+ entryStarted = resolve
+ })
+ ;(filesUtils.uniqueFilePathFromDir as jest.Mock).mockResolvedValueOnce('/tmp/archive.tar.gz')
+ ;(filesUtils.tempFilePath as jest.Mock).mockReturnValueOnce('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ ;(filesUtils.isPathIsDir as jest.Mock).mockImplementationOnce(
+ () =>
+ new Promise<boolean>((resolve) => {
+ resolveEntry = resolve
+ entryStarted()
+ })
+ )
+ const space = makeSpace({ realPath: '/data/users/john/files/source.txt' })
+ const dto = {
+ name: 'archive',
+ extension: 'tar.gz',
+ compressInDirectory: false,
+ files: [{ path: '/data/users/john/files/source.txt', name: 'source.txt', rootAlias: null }]
+ } as any
+
+ const compressPromise = service.compress(user, space, dto)
+ await entryStartedPromise
+ archive.emit('error', error)
+ await new Promise<void>((resolve) => setImmediate(resolve))
+
+ expect(filesUtils.removeFiles).not.toHaveBeenCalled()
+ resolveEntry(false)
+ await expect(compressPromise).rejects.toBe(error)
+ expect(archive.file).not.toHaveBeenCalled()
+ expect(archive.finalize).not.toHaveBeenCalled()
+ expect(filesUtils.removeFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive.tar.gz-compress-uuid')
+ })
})
describe('decompress', () => {
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 0d4bf73b..5f3f691e 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -48,6 +48,7 @@ import {
makeTempDir,
moveFiles,
removeFiles,
+ tempFilePath,
touchFile,
uniqueDatedFilePath,
uniqueFilePathFromDir,
@@ -569,9 +570,10 @@ export class FilesManager {
async downloadFromUrl(user: UserModel, space: SpaceEnv, downloadDto: DownloadFileDto): Promise<void> {
this.checkNotTrashRepository(space)
this.logger.log({ tag: this.downloadFromUrl.name, msg: `${downloadDto.url}` })
- const rPath = await uniqueFilePathFromDir(space.realPath)
+ const dstPath = await uniqueFilePathFromDir(space.realPath)
+ const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-download-`)
const dbFile = space.dbFile
- dbFile.path = path.join(dirName(dbFile.path), fileName(space.realPath))
+ dbFile.path = path.join(dirName(dbFile.path), fileName(dstPath))
// create lock
const [ok, fileLock] = await this.filesLockManager.create(user, dbFile, SERVER_NAME, DEPTH.RESOURCE)
@@ -580,15 +582,18 @@ export class FilesManager {
}
try {
- await new DownloadFile(this.http).download(downloadDto, rPath, { space: space })
+ await new DownloadFile(this.http).download(downloadDto, tmpPath, { space: space, publishedPath: dstPath })
+ await moveFiles(tmpPath, dstPath)
} catch (e) {
- await removeFiles(rPath).catch((err: Error) => this.logger.error({ tag: this.downloadFromUrl.name, msg: `unable to remove ${rPath} : ${err}` }))
+ await removeFiles(tmpPath).catch((err: Error) =>
+ this.logger.error({ tag: this.downloadFromUrl.name, msg: `unable to remove ${tmpPath} : ${err}` })
+ )
throw e
} finally {
// release lock
await this.filesLockManager.removeLock(fileLock.key)
}
- FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: rPath })
+ FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })
}
async compress(user: UserModel, space: SpaceEnv, dto: CompressFileDto): Promise<void> {
@@ -600,6 +605,7 @@ export class FilesManager {
const srcPath = dirName(space.realPath)
const archiveExt = dto.name.endsWith(dto.extension) ? '' : `.${dto.extension}`
const dstPath = await uniqueFilePathFromDir(path.join(dto.compressInDirectory ? srcPath : user.tasksPath, `${dto.name}${archiveExt}`))
+ const tmpPath = tempFilePath(user.tmpPath, `${fileName(dstPath)}-compress-`)
// avoid using ZIP here because it can trigger high memory usage.
const archive: Archiver = archiver(TAR_EXTENSION, {
gzip: dto.extension === TAR_GZ_EXTENSION,
@@ -620,30 +626,46 @@ export class FilesManager {
}
if (space.task?.cacheKey) {
space.task.props.compressInDirectory = dto.compressInDirectory
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath)
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)
}
// do
+ let aborted = false
+ let pipePromise: Promise<void> | undefined
+ let entriesPromise: Promise<void> | undefined
try {
- const dstStream = fs.createWriteStream(dstPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- const pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream
-
- for (const f of dto.files) {
- if (await isPathIsDir(f.path)) {
- archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)
- } else {
- archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })
+ const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ pipePromise = pipeline(archive, dstStream) // handle archive errors + write stream
+ entriesPromise = (async () => {
+ for (const f of dto.files) {
+ const isDir = await isPathIsDir(f.path)
+ if (aborted) return
+ if (isDir) {
+ archive.directory(f.path, dto.files.length > 1 ? fileName(f.path) : false)
+ } else {
+ archive.file(f.path, { name: f.rootAlias ? f.name : fileName(f.path) })
+ }
}
- }
-
- const finalizePromise = archive.finalize()
- await Promise.all([finalizePromise, pipePromise])
+ // The pipeline is the completion signal: finalize() may remain pending after abort().
+ if (!aborted) {
+ void archive.finalize().catch(() => undefined)
+ }
+ })()
+ await Promise.all([entriesPromise, pipePromise])
+ await moveFiles(tmpPath, dstPath)
+ } catch (e) {
+ aborted = true
+ archive.abort()
+ archive.destroy()
+ await Promise.allSettled([entriesPromise, pipePromise].filter((promise): promise is Promise<void> => !!promise))
+ await removeFiles(tmpPath).catch((err: Error) => this.logger.error({ tag: this.compress.name, msg: `unable to remove ${tmpPath} : ${err}` }))
+ throw e
} finally {
if (fileLock) {
await this.filesLockManager.removeLock(fileLock.key)
}
- // emit file event
- FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })
}
+ // emit file event
+ FileEvent.emit('event', { user, space, action: ACTION.ADD, rPath: dstPath })
}
async decompress(user: UserModel, space: SpaceEnv): Promise<void> {
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
index e8c86611..36de13e7 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.spec.ts
@@ -86,7 +86,95 @@ describe(FilesTasksManager.name, () => {
FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')
- expect(startWatchSpy).toHaveBeenCalledWith(space, FILE_OPERATION.DECOMPRESS, '/tmp/archive-extract', 'files/personal', 'archive')
+ expect(startWatchSpy).toHaveBeenCalledWith(
+ space,
+ FILE_OPERATION.DECOMPRESS,
+ '/tmp/archive-extract',
+ 'files/personal',
+ 'archive',
+ '/files/archive'
+ )
+ })
+
+ it('should watch the published archive after the temporary archive has been moved', async () => {
+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)
+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const space = { task: { cacheKey: 'task-1', props: {} } } as any
+
+ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
+ expect(fileSizeSpy).toHaveBeenCalledWith('/files/archive.tgz')
+ expect(space.task.props.size).toBe(42)
+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })
+ })
+
+ it('should retry the published archive when the temporary archive is moved before reading it', async () => {
+ const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })
+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)
+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)
+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const space = { task: { cacheKey: 'task-1', props: {} } } as any
+
+ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
+ expect(fileSizeSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
+ expect(fileSizeSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
+ expect(space.task.props.size).toBe(42)
+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })
+ })
+
+ it('should retry the published directory when the temporary directory is moved while reading it', async () => {
+ const pathExistsSpy = jest.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ const countDirEntriesSpy = jest
+ .spyOn(filesUtils, 'countDirEntries')
+ .mockResolvedValueOnce({ files: 0, directories: 0 })
+ .mockResolvedValueOnce({ files: 2, directories: 1 })
+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const space = { task: { cacheKey: 'task-1', props: {} } } as any
+
+ await (filesTasksManager as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')
+
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')
+ expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')
+ expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
+ expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')
+ expect(space.task.props).toEqual({ files: 2, directories: 1 })
+ expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1 })
+ })
+
+ it('should silently skip watcher update while temporary and published paths are missing', async () => {
+ jest.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)
+ const fileSizeSpy = jest.spyOn(filesUtils, 'fileSize')
+ const loggerErrorSpy = jest.spyOn((filesTasksManager as any).logger, 'error')
+ const stopWatchSpy = jest.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)
+ const updateTaskSpy = jest.spyOn(filesTasksManager as any, 'updateTask').mockResolvedValueOnce(undefined)
+ const space = { task: { cacheKey: 'task-1', props: {} } } as any
+
+ await (filesTasksManager as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
+
+ expect(loggerErrorSpy).not.toHaveBeenCalled()
+ expect(stopWatchSpy).not.toHaveBeenCalled()
+ expect(updateTaskSpy).not.toHaveBeenCalled()
+ expect(fileSizeSpy).not.toHaveBeenCalled()
+ })
+
+ it('should stop a watcher without staging when its path is missing', async () => {
+ const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })
+ jest.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)
+ const loggerErrorSpy = jest.spyOn((filesTasksManager as any).logger, 'error')
+ const stopWatchSpy = jest.spyOn(filesTasksManager as any, 'stopWatch').mockResolvedValueOnce(undefined)
+ const space = { task: { cacheKey: 'task-1', props: {} } } as any
+
+ await (filesTasksManager as any).updateCompressTask(space, '/files/archive.tgz')
+
+ expect(loggerErrorSpy).toHaveBeenCalled()
+ expect(stopWatchSpy).toHaveBeenCalledWith('task-1')
})
it('should create a task and mark it as success when the async method resolves', async () => {
diff --git a/backend/src/applications/files/services/files-tasks-manager.service.ts b/backend/src/applications/files/services/files-tasks-manager.service.ts
index bb03875b..53a7cdc2 100644
--- a/backend/src/applications/files/services/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/files-tasks-manager.service.ts
@@ -11,7 +11,7 @@ import { UserModel } from '../../users/models/user.model'
import { CACHE_TASK_PREFIX, CACHE_TASK_TTL } from '../constants/cache'
import { FILE_OPERATION } from '../constants/operations'
import { FileTask, FileTaskProps, FileTaskStatus } from '../models/file-task'
-import { countDirEntries, dirName, fileName, fileSize, isPathIsDir, removeFiles } from '../utils/files'
+import { countDirEntries, dirName, fileName, fileSize, isPathExists, isPathIsDir, removeFiles } from '../utils/files'
import { SendFile } from '../utils/send-file'
import { FilesMethods } from './files-methods.service'
import { FileTaskEvent } from '../events/file-events'
@@ -28,7 +28,7 @@ export class FilesTasksManager {
private readonly filesMethods: FilesMethods
) {
FileTaskEvent.on('startWatch', async (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string) =>
- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath))
+ this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)
)
}
@@ -144,7 +144,14 @@ export class FilesTasksManager {
}
}
- private async startWatch(space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, taskPath?: string, taskName = fileName(rPath)): Promise<void> {
+ private async startWatch(
+ space: SpaceEnv,
+ taskType: FILE_OPERATION,
+ rPath: string,
+ taskPath?: string,
+ taskName = fileName(rPath),
+ publishedPath?: string
+ ): Promise<void> {
if (!space.task?.cacheKey || space.task.cacheKey in this.tasksWatcher) return
this.logger.verbose({ tag: this.startWatch.name, msg: `${space.task.cacheKey}` })
this.updateTask(space.task.cacheKey, space.task?.props, {
@@ -153,13 +160,13 @@ export class FilesTasksManager {
}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
switch (taskType) {
case FILE_OPERATION.COMPRESS:
- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateCompressTask(space, rPath), this.watchInterval)
+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateCompressTask(space, rPath, publishedPath), this.watchInterval)
return
case FILE_OPERATION.DECOMPRESS:
- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDecompressTask(space, rPath), this.watchInterval)
+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDecompressTask(space, rPath, publishedPath), this.watchInterval)
return
case FILE_OPERATION.DOWNLOAD:
- this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDownloadTask(space, rPath), this.watchInterval)
+ this.tasksWatcher[space.task.cacheKey] = setInterval(async () => this.updateDownloadTask(space, rPath, publishedPath), this.watchInterval)
return
case FILE_OPERATION.COPY:
case FILE_OPERATION.MOVE:
@@ -178,9 +185,11 @@ export class FilesTasksManager {
delete this.tasksWatcher[cacheKey]
}
- private async updateCompressTask(space: SpaceEnv, rPath: string): Promise<void> {
+ private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
try {
- space.task.props.size = await fileSize(rPath)
+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
+ if (size === undefined) return
+ space.task.props.size = size
await this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
)
@@ -190,9 +199,11 @@ export class FilesTasksManager {
}
}
- private async updateDecompressTask(space: SpaceEnv, rPath: string): Promise<void> {
+ private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
try {
- space.task.props = await countDirEntries(rPath)
+ const props = await this.readWatchedPath(rPath, publishedPath, countDirEntries)
+ if (props === undefined) return
+ space.task.props = props
this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
)
@@ -202,9 +213,9 @@ export class FilesTasksManager {
}
}
- private async updateDownloadTask(space: SpaceEnv, rPath: string): Promise<void> {
+ private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
try {
- await this.calcSizeAndProgressTask(space, rPath)
+ if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return
this.updateTask(space.task.cacheKey, space.task.props).catch((e: Error) =>
this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
)
@@ -230,10 +241,28 @@ export class FilesTasksManager {
}
}
- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string) {
- space.task.props.size = await fileSize(rPath)
+ private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {
+ const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
+ if (size === undefined) return false
+ space.task.props.size = size
if (space.task.props.totalSize) {
space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize
}
+ return true
+ }
+
+ private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {
+ if (!publishedPath) return read(rPath)
+ if (await isPathExists(rPath)) {
+ try {
+ const value = await read(rPath)
+ // The temporary path may be moved atomically while it is being read.
+ if (await isPathExists(rPath)) return value
+ } catch (e) {
+ // Retry the published path only when the temporary path disappeared during publication.
+ if (e?.code !== 'ENOENT') throw e
+ }
+ }
+ if (await isPathExists(publishedPath)) return read(publishedPath)
}
}
diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.ts
index 77212f55..3e2ba08e 100644
--- a/backend/src/applications/files/utils/download-file.ts
+++ b/backend/src/applications/files/utils/download-file.ts
@@ -37,12 +37,12 @@ export class DownloadFile {
async download(
downloadDto: DownloadFileDto,
dstPath: string,
- options: { allowPrivateIP?: boolean; space?: SpaceEnv; getContentInfo: true; maxSize?: number }
+ options: { allowPrivateIP?: boolean; space?: SpaceEnv; publishedPath?: string; getContentInfo: true; maxSize?: number }
): Promise<DownloadFileContentInfo>
async download(
downloadDto: DownloadFileDto,
dstPath: string,
- options?: { allowPrivateIP?: boolean; space?: SpaceEnv; getContentInfo?: false | undefined; maxSize?: number }
+ options?: { allowPrivateIP?: boolean; space?: SpaceEnv; publishedPath?: string; getContentInfo?: false | undefined; maxSize?: number }
): Promise<void>
async download(downloadDto: DownloadFileDto, dstPath: string, options?: DownloadFileOptions): Promise<void | DownloadFileContentInfo> {
const identityEncodingConfig = { decompress: false, headers: { 'Accept-Encoding': 'identity' } }
@@ -67,7 +67,7 @@ export class DownloadFile {
if (maxSize === null || (contentLength === null && options?.space)) {
throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR_MESSAGES.DOWNLOAD_INVALID_CONTENT_LENGTH)
}
- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, dstPath)
+ if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath, dstPath)
// The HEAD request resolved redirects; the GET must target that final URL directly.
const { response: getRes } = await this.request(
@@ -173,14 +173,18 @@ export class DownloadFile {
return Number.isSafeInteger(contentLength) ? contentLength : null
}
- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, dstPath: string): void {
+ private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string, dstPath: string): void {
if (!space) return
if (space.willExceedQuota(contentLength)) {
throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)
}
if (space.task?.cacheKey) {
space.task.props.totalSize = contentLength
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)
+ if (publishedPath === dstPath) {
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)
+ } else {
+ FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, publishedPath, dstPath)
+ }
}
}
}
diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.ts
index a9cfe7c8..262f55ea 100644
--- a/backend/src/applications/files/utils/files.spec.ts
+++ b/backend/src/applications/files/utils/files.spec.ts
@@ -5,7 +5,7 @@ import path from 'node:path'
import { Readable } from 'node:stream'
import { FileError } from '../models/file-error'
import { FILE_ERROR_MESSAGES } from './errors'
-import { isPathInside, makeTempDir, writeFromStream } from './files'
+import { isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'
describe(isPathInside.name, () => {
const basePath = path.join(path.sep, 'tmp', 'output')
@@ -88,3 +88,21 @@ describe(makeTempDir.name, () => {
await expect(access(secondPath)).resolves.toBeUndefined()
})
})
+
+describe(tempFilePath.name, () => {
+ it('returns distinct paths with the requested parent and prefix', () => {
+ const parentPath = path.join(path.sep, 'tmp', 'user')
+ const firstPath = tempFilePath(parentPath, 'archive-compress-')
+ const secondPath = tempFilePath(parentPath, 'archive-compress-')
+
+ expect(firstPath).not.toBe(secondPath)
+ expect(path.dirname(firstPath)).toBe(parentPath)
+ expect(path.basename(firstPath)).toMatch(/^archive-compress-/)
+ })
+
+ it('keeps paths with traversal prefixes inside the requested parent', () => {
+ const parentPath = path.join(path.sep, 'tmp', 'user')
+
+ expect(path.dirname(tempFilePath(parentPath, path.join('..', 'archive-')))).toBe(parentPath)
+ })
+})
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index 8f267acd..39e6c791 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -99,6 +99,10 @@ export async function makeTempDir(parentPath: string, prefix: string): Promise<s
return fs.mkdtemp(path.join(parentPath, prefix))
}
+export function tempFilePath(parentPath: string, prefix: string): string {
+ return path.join(parentPath, `${path.basename(prefix)}${crypto.randomUUID()}`)
+}
+
export function getMimeType(fPath: string, isDir: boolean): string {
if (isDir) {
return 'directory'