Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

feat(backend:files): track download, compression and extraction progress

johavencaa6a9285800
diff --git a/backend/src/applications/files/interfaces/download-file.interface.ts b/backend/src/applications/files/interfaces/download-file.interface.ts
index acd0e0c4..7da6962f 100644
--- a/backend/src/applications/files/interfaces/download-file.interface.ts
+++ b/backend/src/applications/files/interfaces/download-file.interface.ts
@@ -12,5 +12,6 @@ export interface DownloadFileOptions {
publishedPath?: string
getContentInfo?: boolean
maxSize?: number
+ onProgress?: (bytes: number) => void
signal?: AbortSignal
}
diff --git a/backend/src/applications/files/interfaces/file-event.interface.ts b/backend/src/applications/files/interfaces/file-event.interface.ts
index b90bab84..3061c6a4 100644
--- a/backend/src/applications/files/interfaces/file-event.interface.ts
+++ b/backend/src/applications/files/interfaces/file-event.interface.ts
@@ -1,10 +1,9 @@
import type { SpaceEnv } from '../../spaces/models/space-env.model'
-import type { FILE_OPERATION } from '../constants/operations'
import type { ACTION } from '../../../common/constants'
import type { UserModel } from '../../users/models/user.model'
export interface FileTaskEventEmit {
- startWatch: [space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string]
+ startWatch: [space: SpaceEnv, rPath: string]
}
export interface FileEventType {
diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.ts
index 82b5efc9..16299c5f 100644
--- a/backend/src/applications/files/interfaces/file-task.interface.ts
+++ b/backend/src/applications/files/interfaces/file-task.interface.ts
@@ -9,7 +9,7 @@ export interface FileTaskTransferOptions {
beforeCommit?: () => Promise<void>
cacheKey: string
crossDevice?: boolean
- onTransferStart?: (temporaryPath: string) => void
+ onTransferStart?: () => void
onProgress?: (bytes: number) => void
overwrite?: boolean
signal: AbortSignal
@@ -20,3 +20,9 @@ export interface FileTaskCopyTaskOptions extends FileTaskTransferOptions {
preserveTimestamps?: boolean
recursive?: boolean
}
+
+export interface FileTaskExtractionEntry {
+ path: string
+ isDirectory: boolean
+ size: number
+}
diff --git a/backend/src/applications/files/services/files-manager.service.spec.ts b/backend/src/applications/files/services/files-manager.service.spec.ts
index a80f6b00..33f8280f 100644
--- a/backend/src/applications/files/services/files-manager.service.spec.ts
+++ b/backend/src/applications/files/services/files-manager.service.spec.ts
@@ -15,7 +15,6 @@ import * as spacesPathUtils from '../../spaces/utils/paths'
import * as spacesPermsUtils from '../../spaces/utils/permissions'
import { DEPTH } from '../../webdav/constants/webdav'
import { ACTION } from '../../../common/constants'
-import { FILE_OPERATION } from '../constants/operations'
import { DownloadFileDto } from '../dto/file-operations.dto'
import { FileEvent, FileTaskEvent } from '../events/file-events'
import { FileError, SourceCleanupError } from '../models/file-error'
@@ -43,7 +42,7 @@ vi.mock('node:dns/promises', () => ({
describe(FilesManager.name, () => {
let service: FilesManager
- let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock }
+ let filesTasksTransfer: { copy: Mock; move: Mock; delete: Mock; createByteProgressHandler: Mock; createExtractionProgressHandler: Mock }
let http: { axiosRef: Mock }
const lookupMock = lookup as Mock
let filesQueries: { moveFiles: Mock; deleteFiles: Mock }
@@ -172,7 +171,13 @@ describe(FilesManager.name, () => {
}
),
move: vi.fn().mockResolvedValue(undefined),
- delete: vi.fn().mockResolvedValue(undefined)
+ delete: vi.fn().mockResolvedValue(undefined),
+ createByteProgressHandler: vi.fn((space) =>
+ vi.fn((bytes: number) => {
+ space.task.props.size = (space.task.props.size || 0) + bytes
+ })
+ ),
+ createExtractionProgressHandler: vi.fn().mockReturnValue(vi.fn())
}
filesLockManager = {
create: vi.fn().mockResolvedValue([true, { key: 'lock-1' }]),
@@ -1054,17 +1059,19 @@ describe(FilesManager.name, () => {
await service.downloadFromUrl(user, space, { url: 'https://example.org/file.txt' })
- expect(space.task.props.totalSize).toBe(55)
- expect(taskEmitSpy).toHaveBeenCalledWith(
- 'startWatch',
- space,
- FILE_OPERATION.DOWNLOAD,
- '/tmp/download.txt',
- '/data/users/john/tmp/tasks/.task-1-download.txt'
- )
+ expect(space.task.props).toMatchObject({ progress: 1, size: 0, totalSize: 55 })
+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/download.txt')
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-1', '/tmp/download.txt')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
- expect(filesUtils.writeFromStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', expect.anything(), 0, 55)
+ expect(filesUtils.writeFromStream).toHaveBeenCalledWith(
+ '/data/users/john/tmp/tasks/.task-1-download.txt',
+ expect.anything(),
+ 0,
+ 55,
+ undefined,
+ expect.any(Function)
+ )
+ expect(filesTasksTransfer.createByteProgressHandler).toHaveBeenCalledWith(space)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-1-download.txt', '/tmp/download.txt')
expect(filesLockManager.create).toHaveBeenCalledWith(
user,
@@ -1129,6 +1136,9 @@ describe(FilesManager.name, () => {
describe('compress', () => {
it('should archive files and emit events', async () => {
const archive = createArchiveMock()
+ archive.finalize.mockImplementationOnce(async () => {
+ archive.end('content')
+ })
vi.mocked(filesUtils.uniqueFilePathFromDir).mockResolvedValueOnce('/tmp/archive.tar.gz')
vi.mocked(filesUtils.isPathIsDir).mockImplementation(async (p: string) => p.endsWith('/dir'))
const space = makeSpace({ realPath: '/data/users/john/files/source.txt', task: { cacheKey: 'task-c', props: {} } })
@@ -1149,18 +1159,13 @@ describe(FilesManager.name, () => {
expect(archive.directory).toHaveBeenCalled()
expect(archive.file).toHaveBeenCalled()
expect(archive.finalize).toHaveBeenCalled()
- expect(taskEmitSpy).toHaveBeenCalledWith(
- 'startWatch',
- space,
- FILE_OPERATION.COMPRESS,
- '/tmp/archive.tar.gz',
- '/data/users/john/tmp/tasks/.task-c-archive.tar.gz'
- )
+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/tmp/archive.tar.gz')
expect(taskUtils.taskTemporaryPath).toHaveBeenCalledWith(user.tasksPath, 'task-c', '/tmp/archive.tar.gz')
expect(filesUtils.tempFilePath).not.toHaveBeenCalled()
expect(fs.createWriteStream).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', {
highWaterMark: expect.any(Number)
})
+ expect(space.task.props.size).toBe(Buffer.byteLength('content'))
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-c-archive.tar.gz', '/tmp/archive.tar.gz')
})
@@ -1314,15 +1319,16 @@ describe(FilesManager.name, () => {
expect(taskUtils.createTaskTemporaryDir).toHaveBeenCalledWith(user.tasksPath, 'task-d', '/data/users/john/files/archive')
expect(filesUtils.makeTempDir).not.toHaveBeenCalled()
- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/tasks/.task-d-archive', undefined, undefined)
- expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
- expect(taskEmitSpy).toHaveBeenCalledWith(
- 'startWatch',
- space,
- FILE_OPERATION.DECOMPRESS,
- '/data/users/john/files/archive',
- '/data/users/john/tmp/tasks/.task-d-archive'
+ expect(filesTasksTransfer.createExtractionProgressHandler).toHaveBeenCalledWith(space)
+ expect(unzipSpy).toHaveBeenCalledWith(
+ '/data/users/john/files/archive.zip',
+ '/data/users/john/tmp/tasks/.task-d-archive',
+ undefined,
+ undefined,
+ expect.any(Function)
)
+ expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/tasks/.task-d-archive', '/data/users/john/files/archive')
+ expect(taskEmitSpy).toHaveBeenCalledWith('startWatch', space, '/data/users/john/files/archive')
expect(filesLockManager.removeLock).toHaveBeenCalledWith('lock-1')
})
@@ -1340,6 +1346,7 @@ describe(FilesManager.name, () => {
'/data/users/john/tmp/archive-extract-123',
true,
undefined,
+ undefined,
undefined
)
expect(filesUtils.moveFiles).toHaveBeenCalledWith('/data/users/john/tmp/archive-extract-123', '/data/users/john/files/archive')
@@ -1354,7 +1361,13 @@ describe(FilesManager.name, () => {
await service.decompress(user, space)
- expect(unzipSpy).toHaveBeenCalledWith('/data/users/john/files/archive.zip', '/data/users/john/tmp/archive-extract-123', 60, undefined)
+ expect(unzipSpy).toHaveBeenCalledWith(
+ '/data/users/john/files/archive.zip',
+ '/data/users/john/tmp/archive-extract-123',
+ 60,
+ undefined,
+ undefined
+ )
})
it('should remove partial extraction and skip add event on failure', async () => {
diff --git a/backend/src/applications/files/services/files-manager.service.ts b/backend/src/applications/files/services/files-manager.service.ts
index 8997308e..3f86d130 100644
--- a/backend/src/applications/files/services/files-manager.service.ts
+++ b/backend/src/applications/files/services/files-manager.service.ts
@@ -23,7 +23,6 @@ import { DEPTH, LOCK_DEPTH } from '../../webdav/constants/webdav'
import { CACHE_LOCK_FILE_TTL } from '../constants/cache'
import { TAR_EXTENSION, TAR_GZ_EXTENSION } from '../constants/compress'
import { COMPRESSION_EXTENSION, DEFAULT_HIGH_WATER_MARK } from '../constants/files'
-import { FILE_OPERATION } from '../constants/operations'
import { ALL_DOCUMENT_TYPES, DEFAULT_DOCUMENT_TYPES, SAMPLE_PATH_WITHOUT_EXT } from '../constants/samples'
import { CompressFileDto, DownloadFileDto } from '../dto/file-operations.dto'
import { FileDBProps } from '../interfaces/file-db-props.interface'
@@ -36,6 +35,7 @@ import {
copyFileContent,
copyFiles,
createEmptyFile,
+ createProgressTransform,
dirName,
dirSize,
fileName,
@@ -486,7 +486,7 @@ export class FilesManager {
// do
if (isMove) {
let sourceCleanupError: SourceCleanupError | undefined
- if (useTaskTransfer) {
+ if (useTaskTransfer && signal) {
sourceCleanupError = await this.filesTasksTransfer.move(user, srcSpace, dstSpace, overwrite, isDir, signal, () => this.delete(user, dstSpace))
} else {
await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
@@ -604,7 +604,12 @@ export class FilesManager {
}
try {
- await new DownloadFile(this.http).download(downloadDto, tmpPath, { space, publishedPath: dstPath, signal })
+ await new DownloadFile(this.http).download(downloadDto, tmpPath, {
+ space,
+ publishedPath: dstPath,
+ signal,
+ onProgress: isTaskContext ? this.filesTasksTransfer.createByteProgressHandler(space) : undefined
+ })
signal?.throwIfAborted()
await moveFiles(tmpPath, dstPath)
} catch (e) {
@@ -652,7 +657,8 @@ export class FilesManager {
}
if (isTaskContext) {
space.task!.props.compressInDirectory = dto.compressInDirectory
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.COMPRESS, dstPath, tmpPath)
+ space.task!.props.size = 0
+ FileTaskEvent.emit('startWatch', space, dstPath)
}
// do
let aborted = false
@@ -660,7 +666,12 @@ export class FilesManager {
let entriesPromise: Promise<void> | undefined
try {
const dstStream = fs.createWriteStream(tmpPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- pipePromise = pipeline(archive, dstStream, { signal }) // handle archive errors + write stream
+ if (isTaskContext) {
+ const onProgress = this.filesTasksTransfer.createByteProgressHandler(space)
+ pipePromise = pipeline(archive, createProgressTransform(onProgress), dstStream, { signal })
+ } else {
+ pipePromise = pipeline(archive, dstStream, { signal })
+ }
entriesPromise = (async () => {
for (const f of dto.files) {
signal?.throwIfAborted()
@@ -723,13 +734,14 @@ export class FilesManager {
}
fileLock = lock
// tasking
- if (isTaskContext) FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, dstPath, tmpPath)
+ const onEntry = isTaskContext ? this.filesTasksTransfer.createExtractionProgressHandler(space) : undefined
+ if (isTaskContext) FileTaskEvent.emit('startWatch', space, dstPath)
// do
const maxExtractedSize = space.storageQuota === null ? undefined : Math.max(0, space.storageQuota - space.storageUsage)
if (extension === '.zip') {
- await extractZip(space.realPath, tmpPath, maxExtractedSize, signal)
+ await extractZip(space.realPath, tmpPath, maxExtractedSize, signal, onEntry)
} else {
- await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal)
+ await extractTar(space.realPath, tmpPath, COMPRESSION_EXTENSION.get(extension) === TAR_GZ_EXTENSION, maxExtractedSize, signal, onEntry)
}
signal?.throwIfAborted()
if (await isPathExists(dstPath)) {
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
index becac1ba..30341272 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
@@ -4,7 +4,6 @@ import crypto from 'node:crypto'
import { Cache } from '../../../../infrastructure/cache/cache.service'
import { SpacesManager } from '../../../spaces/services/spaces-manager.service'
import * as filesUtils from '../../utils/files'
-import * as tasksUtils from '../../utils/tasks'
import { FileTaskEvent } from '../../events/file-events'
import { SendFile } from '../../utils/send-file'
import { FILE_OPERATION } from '../../constants/operations'
@@ -121,9 +120,9 @@ describe(FilesTasksManager.name, () => {
it('should unref watchers and clear them on module destroy', async () => {
const cacheKey = 'task-1'
const listenersBeforeDestroy = FileTaskEvent.listenerCount('startWatch')
- const space = { task: { cacheKey, props: {} } } as any
+ const space = { url: 'files/personal/document.txt', task: { cacheKey, props: {} } } as any
cacheStore.set(cacheKey, { props: {} })
- ;(filesTasksWatcher as any).startWatch(space, FILE_OPERATION.COPY, '/files/document.txt')
+ ;(filesTasksWatcher as any).startWatch(space, '/files/document.txt')
const watcher = (filesTasksWatcher as any).tasksWatcher[cacheKey] as NodeJS.Timeout
expect(watcher.hasRef()).toBe(false)
@@ -218,101 +217,13 @@ describe(FilesTasksManager.name, () => {
}
})
- it('should watch a temporary path while keeping the published file name', () => {
+ it('should watch task props while keeping the published file metadata', () => {
const startWatchSpy = vi.spyOn(filesTasksWatcher as any, 'startWatch').mockResolvedValueOnce(undefined)
const space = { url: 'files/personal/archive.zip', task: { cacheKey: 'task-1', props: {} } } as any
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DECOMPRESS, '/files/archive', '/tmp/archive-extract')
+ FileTaskEvent.emit('startWatch', space, '/files/archive')
- expect(startWatchSpy).toHaveBeenCalledWith(
- space,
- FILE_OPERATION.DECOMPRESS,
- '/tmp/archive-extract',
- 'files/personal',
- 'archive',
- '/files/archive'
- )
- })
-
- it('should watch the published archive after the temporary archive has been moved', async () => {
- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(true)
- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(42)
- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
- const space = { task: { cacheKey: 'task-1', props: {} } } as any
-
- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
-
- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
- expect(fileSizeSpy).toHaveBeenCalledWith('/files/archive.tgz')
- expect(space.task.props.size).toBe(42)
- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })
- })
-
- it('should retry the published archive when the temporary archive is moved before reading it', async () => {
- const missingError = Object.assign(new Error('missing temporary archive'), { code: 'ENOENT' })
- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(true)
- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError).mockResolvedValueOnce(42)
- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
- const space = { task: { cacheKey: 'task-1', props: {} } } as any
-
- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
-
- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
- expect(fileSizeSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-compress-uuid')
- expect(fileSizeSpy).toHaveBeenNthCalledWith(2, '/files/archive.tgz')
- expect(space.task.props.size).toBe(42)
- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { size: 42 })
- })
-
- it('should retry the published directory when the temporary directory is moved while reading it', async () => {
- const pathExistsSpy = vi.spyOn(filesUtils, 'isPathExists').mockResolvedValueOnce(true).mockResolvedValueOnce(false).mockResolvedValueOnce(true)
- const countDirEntriesSpy = vi
- .spyOn(tasksUtils, 'countDirEntriesAndSize')
- .mockResolvedValueOnce({ files: 0, directories: 0, size: 0 })
- .mockResolvedValueOnce({ files: 2, directories: 1, size: 42 })
- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
- const space = { task: { cacheKey: 'task-1', props: {} } } as any
-
- await (filesTasksWatcher as any).updateDecompressTask(space, '/tmp/archive-extract-uuid', '/files/archive')
-
- expect(pathExistsSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
- expect(pathExistsSpy).toHaveBeenNthCalledWith(2, '/tmp/archive-extract-uuid')
- expect(pathExistsSpy).toHaveBeenNthCalledWith(3, '/files/archive')
- expect(countDirEntriesSpy).toHaveBeenNthCalledWith(1, '/tmp/archive-extract-uuid')
- expect(countDirEntriesSpy).toHaveBeenNthCalledWith(2, '/files/archive')
- expect(space.task.props).toEqual({ files: 2, directories: 1, size: 42 })
- expect(updateTaskSpy).toHaveBeenCalledWith('task-1', { files: 2, directories: 1, size: 42 })
- })
-
- it('should silently skip watcher update while temporary and published paths are missing', async () => {
- vi.spyOn(filesUtils, 'isPathExists').mockResolvedValue(false)
- const fileSizeSpy = vi.spyOn(filesUtils, 'fileSize')
- const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')
- const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)
- const updateTaskSpy = vi.spyOn(filesTasksWatcher as any, 'updateTask').mockResolvedValueOnce(undefined)
- const space = { task: { cacheKey: 'task-1', props: {} } } as any
-
- await (filesTasksWatcher as any).updateCompressTask(space, '/tmp/archive-compress-uuid', '/files/archive.tgz')
-
- expect(loggerErrorSpy).not.toHaveBeenCalled()
- expect(stopWatchSpy).not.toHaveBeenCalled()
- expect(updateTaskSpy).not.toHaveBeenCalled()
- expect(fileSizeSpy).not.toHaveBeenCalled()
- })
-
- it('should stop a watcher without staging when its path is missing', async () => {
- const missingError = Object.assign(new Error('missing archive'), { code: 'ENOENT' })
- vi.spyOn(filesUtils, 'fileSize').mockRejectedValueOnce(missingError)
- const loggerErrorSpy = vi.spyOn((filesTasksWatcher as any).logger, 'error')
- const stopWatchSpy = vi.spyOn(filesTasksWatcher as any, 'stopWatch').mockResolvedValueOnce(undefined)
- const space = { task: { cacheKey: 'task-1', props: {} } } as any
-
- await (filesTasksWatcher as any).updateCompressTask(space, '/files/archive.tgz')
-
- expect(loggerErrorSpy).toHaveBeenCalled()
- expect(stopWatchSpy).toHaveBeenCalledWith('task-1')
+ expect(startWatchSpy).toHaveBeenCalledWith(space, '/files/archive')
})
it('should not let an in-flight progress update overwrite task completion', async () => {
@@ -433,6 +344,31 @@ describe(FilesTasksManager.name, () => {
})
})
+ it('should finalize fast extraction metrics without waiting for a watcher tick', async () => {
+ const taskId = '77777777-7777-4777-8777-777777777777'
+ vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce(taskId)
+ filesMethods.doWork.mockImplementationOnce(async (_user, taskSpace) => {
+ taskSpace.task.props = { directories: 2, files: 3, size: 120 }
+ })
+ const user = { id: 7 } as any
+ const space = {
+ realPath: '/data/users/john/files/archive.zip',
+ url: 'files/personal/archive.zip'
+ } as any
+
+ await filesTasksManager.createTask(FILE_OPERATION.DECOMPRESS, user, space, null, 'doWork')
+ await flushPromises()
+
+ expect(cacheStore.get(`${CACHE_TASK_PREFIX}-7-${taskId}`)).toMatchObject({
+ status: FileTaskStatus.SUCCESS,
+ props: {
+ directories: 2,
+ files: 3,
+ size: 120
+ }
+ })
+ })
+
it('should create a task and mark it as error when the async method fails', async () => {
vi.spyOn(crypto, 'randomUUID').mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
filesMethods.doWork.mockRejectedValueOnce(new Error('operation failed'))
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
index 5d096321..b0458869 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
@@ -206,7 +206,7 @@ export class FilesTasksManager implements OnModuleDestroy {
taskPromise
.then(async (data: any) => {
this.logger.debug({ tag: this.runTask.name, msg: `${task.task.name} : ${task.method} done` })
- const finalProps = await this.getFinalTransferProps(task, data)
+ const finalProps = await this.getFinalTaskProps(task, data)
this.completeTask(task.user.id, task.cacheKey, FileTaskStatus.SUCCESS, data, finalProps).catch((e: Error) =>
this.logger.warn({ tag: this.runTask.name, msg: `${e}` })
)
@@ -231,7 +231,10 @@ export class FilesTasksManager implements OnModuleDestroy {
}
}
- private async getFinalTransferProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {
+ private async getFinalTaskProps(task: FileTaskQueueItem, result: any): Promise<FileTaskProps | undefined> {
+ if (task.task.type === FILE_OPERATION.DOWNLOAD || task.task.type === FILE_OPERATION.COMPRESS || task.task.type === FILE_OPERATION.DECOMPRESS) {
+ return task.space.task?.props
+ }
if (task.task.type === FILE_OPERATION.DELETE) {
const props = task.space.task?.props
return {
@@ -248,7 +251,7 @@ export class FilesTasksManager implements OnModuleDestroy {
const props = await this.filesTasksWatcher.getPathProps(publishedSpace.realPath)
return { ...props, progress: 100 }
} catch (e) {
- this.logger.warn({ tag: this.getFinalTransferProps.name, msg: `${e}` })
+ this.logger.warn({ tag: this.getFinalTaskProps.name, msg: `${e}` })
}
}
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
index 30884f9d..e7e06f62 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
@@ -53,9 +53,28 @@ describe(FilesTasksTransfer.name, () => {
await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)
})
+ it('tracks extracted entries and implicit parent directories', () => {
+ const space = {
+ task: { cacheKey, props: { progress: 1 } }
+ } as any
+ const onEntry = service.createExtractionProgressHandler(space)
+
+ onEntry({ path: 'docs/guides/readme.txt', isDirectory: false, size: 7 })
+ onEntry({ path: 'docs/images/', isDirectory: true, size: 0 })
+ onEntry({ path: 'docs/guides/second.txt', isDirectory: false, size: 5 })
+ onEntry({ path: 'docs/guides/readme.txt', isDirectory: false, size: 9 })
+
+ expect(space.task.props).toEqual({
+ progress: 1,
+ files: 2,
+ directories: 3,
+ size: 14
+ })
+ })
+
it('publishes a copied file only after the commit hook', async () => {
const signal = new AbortController().signal
- let temporaryPath = ''
+ const temporaryPath = path.join(stagingDir, `.${cacheKey}-destination.txt`)
let transferredBytes = 0
await writeFile(srcPath, 'content')
@@ -65,9 +84,7 @@ describe(FilesTasksTransfer.name, () => {
await expect(readFile(temporaryPath, 'utf8')).resolves.toBe('content')
},
cacheKey,
- onTransferStart: (value: string) => {
- temporaryPath = value
- },
+ onTransferStart: vi.fn(),
onProgress: (bytes: number) => {
transferredBytes += bytes
},
@@ -95,7 +112,7 @@ describe(FilesTasksTransfer.name, () => {
it('copies directly to a destination on another device', async () => {
const signal = new AbortController().signal
- let watchedPath = ''
+ let transferStarted = false
let destinationPrepared = false
await writeFile(srcPath, 'content')
mockCrossDevice()
@@ -105,15 +122,15 @@ describe(FilesTasksTransfer.name, () => {
destinationPrepared = true
},
cacheKey,
- onTransferStart: (value: string) => {
+ onTransferStart: () => {
expect(destinationPrepared).toBe(true)
- watchedPath = value
+ transferStarted = true
},
signal,
stagingDir
})
- expect(watchedPath).toBe(dstPath)
+ expect(transferStarted).toBe(true)
await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
await expect(readdir(stagingDir)).resolves.toEqual([])
})
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
index 092f299a..a0de7d49 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
@@ -2,16 +2,14 @@ import { Injectable } from '@nestjs/common'
import { createReadStream, createWriteStream } from 'node:fs'
import fs from 'node:fs/promises'
import path from 'node:path'
-import { Transform } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { UserModel } from '../../../users/models/user.model'
import { SpaceEnv } from '../../../spaces/models/space-env.model'
import { DEFAULT_HIGH_WATER_MARK } from '../../constants/files'
-import { FILE_OPERATION } from '../../constants/operations'
import { FileTaskEvent } from '../../events/file-events'
-import { fileSize, isPathExists, moveFiles, removeFiles } from '../../utils/files'
-import { countDirEntriesAndSize, isCrossDeviceMove, taskTemporaryPath } from '../../utils/tasks'
-import { FileTaskCopyTaskOptions, FileTaskTransferOptions } from '../../interfaces/file-task.interface'
+import type { FileTaskCopyTaskOptions, FileTaskExtractionEntry, FileTaskTransferOptions } from '../../interfaces/file-task.interface'
+import { createProgressTransform, fileSize, isCrossDevice, isPathExists, moveFiles, removeFiles } from '../../utils/files'
+import { countDirEntriesAndSize, taskTemporaryPath } from '../../utils/tasks'
import { SourceCleanupError } from '../../models/file-error'
@Injectable()
@@ -30,8 +28,8 @@ export class FilesTasksTransfer {
await this.copyAbortable(srcSpace.realPath, dstSpace.realPath, {
beforeCommit: this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination),
cacheKey: srcSpace.task!.cacheKey,
- onProgress: (bytes) => this.updateProgress(srcSpace, bytes),
- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.COPY, dstSpace.realPath, watchPath),
+ onProgress: this.createByteProgressHandler(srcSpace),
+ onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),
overwrite,
recursive,
signal,
@@ -45,24 +43,17 @@ export class FilesTasksTransfer {
dstSpace: SpaceEnv,
overwrite: boolean,
isDir: boolean,
- signal: AbortSignal | undefined,
+ signal: AbortSignal,
deleteDestination: () => Promise<void>
): Promise<SourceCleanupError | undefined> {
await this.initializeTaskProps(srcSpace, isDir)
const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)
- if (!signal) {
- // Same-device moves stay atomic and do not copy bytes.
- this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath)
- await beforeCommit?.()
- await moveFiles(srcSpace.realPath, dstSpace.realPath, overwrite)
- return
- }
return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {
beforeCommit,
cacheKey: srcSpace.task!.cacheKey,
crossDevice: true,
- onProgress: (bytes) => this.updateProgress(srcSpace, bytes),
- onTransferStart: (watchPath) => this.startTransferTaskWatch(srcSpace, FILE_OPERATION.MOVE, dstSpace.realPath, watchPath),
+ onProgress: this.createByteProgressHandler(srcSpace),
+ onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),
overwrite,
signal,
stagingDir: user.tasksPath
@@ -80,7 +71,7 @@ export class FilesTasksTransfer {
await this.initializeTaskProps(space, isDir)
if (!signal) {
// Moving to trash on the same device stays atomic and does not copy bytes.
- this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile)
+ this.startTransferTaskWatch(space, trashFile)
await prepareDestination()
await moveFiles(space.realPath, trashFile, true)
return
@@ -89,8 +80,8 @@ export class FilesTasksTransfer {
beforeCommit: prepareDestination,
cacheKey: space.task!.cacheKey,
crossDevice: true,
- onProgress: (bytes) => this.updateProgress(space, bytes),
- onTransferStart: (watchPath) => this.startTransferTaskWatch(space, FILE_OPERATION.DELETE, trashFile, watchPath),
+ onProgress: this.createByteProgressHandler(space),
+ onTransferStart: () => this.startTransferTaskWatch(space, trashFile),
overwrite: true,
signal,
stagingDir: user.tasksPath
@@ -108,11 +99,35 @@ export class FilesTasksTransfer {
}
}
- private updateProgress(space: SpaceEnv, bytes: number): void {
- const props = space.task!.props
- props.size = Math.min((props.size || 0) + bytes, props.totalSize || Number.MAX_SAFE_INTEGER)
- if (props.totalSize) {
- props.progress = Math.min((100 * props.size) / props.totalSize, 100)
+ createByteProgressHandler(space: SpaceEnv): (bytes: number) => void {
+ return (bytes) => {
+ const props = space.task!.props
+ props.size = Math.min((props.size ?? 0) + bytes, props.totalSize ?? Number.MAX_SAFE_INTEGER)
+ if (props.totalSize) {
+ props.progress = Math.min((100 * props.size) / props.totalSize, 100)
+ }
+ }
+ }
+
+ createExtractionProgressHandler(space: SpaceEnv): (entry: FileTaskExtractionEntry) => void {
+ const directories = new Set<string>()
+ const files = new Map<string, number>()
+ space.task!.props = { ...space.task!.props, files: 0, directories: 0, size: 0 }
+ return (entry) => {
+ const parts = entry.path.split(/[/\\]/).filter((part) => part && part !== '.')
+ const directoryParts = entry.isDirectory ? parts : parts.slice(0, -1)
+ for (let index = 1; index <= directoryParts.length; index++) {
+ directories.add(directoryParts.slice(0, index).join('/'))
+ }
+ const props = space.task!.props
+ props.directories = directories.size
+ if (!entry.isDirectory && parts.length) {
+ const filePath = parts.join('/')
+ const previousSize = files.get(filePath) ?? 0
+ files.set(filePath, entry.size)
+ props.files = files.size
+ props.size = (props.size ?? 0) - previousSize + entry.size
+ }
}
}
@@ -130,8 +145,8 @@ export class FilesTasksTransfer {
}
}
- private startTransferTaskWatch(space: SpaceEnv, operation: FILE_OPERATION, publishedPath: string, watchPath?: string): void {
- FileTaskEvent.emit('startWatch', space, operation, publishedPath, watchPath)
+ private startTransferTaskWatch(space: SpaceEnv, publishedPath: string): void {
+ FileTaskEvent.emit('startWatch', space, publishedPath)
}
private async copyAbortable(srcPath: string, dstPath: string, options: FileTaskCopyTaskOptions): Promise<void> {
@@ -148,7 +163,7 @@ export class FilesTasksTransfer {
} = options
const temporaryPath = taskTemporaryPath(stagingDir, cacheKey, dstPath)
await fs.mkdir(stagingDir, { recursive: true })
- const copyDirectlyToDestination = await isCrossDeviceMove(stagingDir, dstPath)
+ const copyDirectlyToDestination = await isCrossDevice(stagingDir, dstPath)
if (copyDirectlyToDestination) {
let transferStarted = false
@@ -157,7 +172,7 @@ export class FilesTasksTransfer {
await beforeCommit?.()
signal.throwIfAborted()
await this.prepareDestination(dstPath, overwrite)
- onTransferStart?.(dstPath)
+ onTransferStart?.()
transferStarted = true
await this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)
} catch (e) {
@@ -170,7 +185,7 @@ export class FilesTasksTransfer {
}
try {
- onTransferStart?.(temporaryPath)
+ onTransferStart?.()
await this.copyEntry(srcPath, temporaryPath, recursive, preserveTimestamps, signal, onProgress)
signal.throwIfAborted()
await beforeCommit?.()
@@ -184,7 +199,7 @@ export class FilesTasksTransfer {
private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {
const { beforeCommit, cacheKey, onProgress, onTransferStart, overwrite = false, signal, stagingDir } = options
- const crossDevice = options.crossDevice ?? (await isCrossDeviceMove(srcPath, dstPath))
+ const crossDevice = options.crossDevice ?? (await isCrossDevice(srcPath, dstPath))
if (!crossDevice) {
signal.throwIfAborted()
await beforeCommit?.()
@@ -222,13 +237,7 @@ export class FilesTasksTransfer {
const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })
if (onProgress) {
- const progress = new Transform({
- transform(chunk: Buffer, _encoding, callback) {
- onProgress(chunk.length)
- callback(null, chunk)
- }
- })
- await pipeline(src, progress, dst, { signal })
+ await pipeline(src, createProgressTransform(onProgress), dst, { signal })
} else {
await pipeline(src, dst, { signal })
}
diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts
index 9b00d02b..28202507 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.spec.ts
@@ -1,6 +1,5 @@
import { Cache } from '../../../../infrastructure/cache/cache.service'
import { CACHE_TASK_TTL } from '../../constants/cache'
-import { FILE_OPERATION } from '../../constants/operations'
import { FileTaskEvent } from '../../events/file-events'
import { FileTaskStatus } from '../../models/file-task'
import * as filesUtils from '../../utils/files'
@@ -65,22 +64,21 @@ describe(FilesTasksWatcher.name, () => {
it('starts a download watcher with the published task metadata', async () => {
const space = makeSpace()
const updateTask = vi.spyOn(service as any, 'updateTask').mockResolvedValue(undefined)
- const updateDownloadTask = vi.spyOn(service as any, 'updateDownloadTask').mockResolvedValue(undefined)
const watchTask = vi.spyOn(service as any, 'watchTask').mockImplementation((_cacheKey: string, update: () => Promise<void>) => void update())
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, '/files/report.txt', '/tmp/report.part')
+ FileTaskEvent.emit('startWatch', space, '/files/report.txt')
await flushPromises()
expect(updateTask).toHaveBeenCalledWith('task-1', {}, { name: 'report.txt', path: 'files/personal' })
expect(watchTask).toHaveBeenCalledWith('task-1', expect.any(Function))
- expect(updateDownloadTask).toHaveBeenCalledWith(space, '/tmp/report.part', '/files/report.txt')
+ expect(updateTask).toHaveBeenCalledWith('task-1', space.task.props)
})
it('does not start duplicate watchers for the same task', () => {
const space = makeSpace()
const watchTask = vi.spyOn(service as any, 'watchTask')
;(service as any).tasksWatcher['task-1'] = setInterval(() => undefined, 1000)
- ;(service as any).startWatch(space, FILE_OPERATION.COPY, '/files/report.txt')
+ ;(service as any).startWatch(space, '/files/report.txt')
expect(watchTask).not.toHaveBeenCalled()
})
@@ -161,21 +159,4 @@ describe(FilesTasksWatcher.name, () => {
service.endFinalization('task-1')
expect((service as any).finalizingTasks.has('task-1')).toBe(false)
})
-
- it('falls back to the published file and updates download progress', async () => {
- const space = makeSpace('task-1', { size: 0, totalSize: 200 })
- cacheStore.set('task-1', {
- props: { size: 0, totalSize: 200 },
- status: FileTaskStatus.PENDING
- })
- vi.spyOn(filesUtils, 'isPathExists').mockImplementation(async (rPath: string) => rPath === '/files/report.txt')
- vi.spyOn(filesUtils, 'isPathIsDir').mockResolvedValueOnce(false)
- vi.spyOn(filesUtils, 'fileSize').mockResolvedValueOnce(50)
-
- await (service as any).updateDownloadTask(space, '/tmp/report.part', '/files/report.txt')
-
- expect(filesUtils.fileSize).toHaveBeenCalledWith('/files/report.txt')
- expect(space.task.props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })
- expect(cacheStore.get('task-1').props).toMatchObject({ progress: 25, size: 50, totalSize: 200 })
- })
})
diff --git a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts
index 16d09d1d..77c82db4 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-watcher.service.ts
@@ -2,10 +2,9 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'
import { Cache } from '../../../../infrastructure/cache/cache.service'
import { SpaceEnv } from '../../../spaces/models/space-env.model'
import { CACHE_TASK_TTL } from '../../constants/cache'
-import { FILE_OPERATION } from '../../constants/operations'
import { FileTaskEvent } from '../../events/file-events'
import { FileTask, FileTaskProps, FileTaskStatus } from '../../models/file-task'
-import { dirName, fileName, fileSize, isPathExists, isPathIsDir } from '../../utils/files'
+import { dirName, fileName, fileSize, isPathIsDir } from '../../utils/files'
import { countDirEntriesAndSize } from '../../utils/tasks'
@Injectable()
@@ -47,47 +46,23 @@ export class FilesTasksWatcher implements OnModuleDestroy {
return (await isPathIsDir(rPath)) ? countDirEntriesAndSize(rPath) : { size: await fileSize(rPath) }
}
- private readonly onStartWatch = (space: SpaceEnv, taskType: FILE_OPERATION, rPath: string, watchPath?: string): void => {
- this.startWatch(space, taskType, watchPath || rPath, dirName(space.url), fileName(rPath), watchPath ? rPath : undefined)
+ private readonly onStartWatch = (space: SpaceEnv, rPath: string): void => {
+ this.startWatch(space, rPath)
}
- private startWatch(
- space: SpaceEnv,
- taskType: FILE_OPERATION,
- rPath: string,
- taskPath?: string,
- taskName = fileName(rPath),
- publishedPath?: string
- ): void {
+ private startWatch(space: SpaceEnv, rPath: string): void {
const taskContext = space.task
if (!taskContext?.cacheKey || taskContext.cacheKey in this.tasksWatcher) return
const { cacheKey, props } = taskContext
this.updateTask(cacheKey, props, {
- name: taskName,
- path: taskPath
+ name: fileName(rPath),
+ path: dirName(space.url)
}).catch((e: Error) => this.logger.error({ tag: this.startWatch.name, msg: `${e}` }))
- switch (taskType) {
- case FILE_OPERATION.COMPRESS:
- this.watchTask(cacheKey, () => this.updateCompressTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.DECOMPRESS:
- this.watchTask(cacheKey, () => this.updateDecompressTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.DOWNLOAD:
- this.watchTask(cacheKey, () => this.updateDownloadTask(space, rPath, publishedPath))
- return
- case FILE_OPERATION.COPY:
- case FILE_OPERATION.DELETE:
- case FILE_OPERATION.MOVE:
- this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))
- return
- default:
- this.logger.warn({ tag: this.startWatch.name, msg: `unknown task type ${taskType}` })
- }
+ this.watchTask(cacheKey, () => this.updateTask(cacheKey, space.task.props))
}
private watchTask(cacheKey: string, update: () => Promise<void>): void {
- // Directory scans may exceed the interval, so ticks use exhaust semantics.
+ // Cache updates may exceed the interval, so ticks use exhaust semantics.
let updateInProgress = false
const watcher = setInterval(() => {
if (updateInProgress) return
@@ -131,71 +106,4 @@ export class FilesTasksWatcher implements OnModuleDestroy {
private isActiveStatus(status: FileTaskStatus): boolean {
return status === FileTaskStatus.PENDING || status === FileTaskStatus.QUEUED
}
-
- private async updateCompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
- if (size === undefined) return
- space.task.props.size = size
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateCompressTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async updateDecompressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)
- if (props === undefined) return
- space.task.props = props
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateDecompressTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async updateDownloadTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<void> {
- try {
- if (!(await this.calcSizeAndProgressTask(space, rPath, publishedPath))) return
- await this.updateTask(space.task.cacheKey, space.task.props)
- } catch (e) {
- this.logger.error({ tag: this.updateDownloadTask.name, msg: `${e}` })
- this.stopWatch(space.task.cacheKey)
- }
- }
-
- private async calcSizeAndProgressTask(space: SpaceEnv, rPath: string, publishedPath?: string): Promise<boolean> {
- const isDir = await this.readWatchedPath(rPath, publishedPath, isPathIsDir)
- if (isDir === undefined) return false
- if (isDir) {
- const props = await this.readWatchedPath(rPath, publishedPath, countDirEntriesAndSize)
- if (props === undefined) return false
- space.task.props = { ...space.task.props, ...props }
- } else {
- const size = await this.readWatchedPath(rPath, publishedPath, fileSize)
- if (size === undefined) return false
- space.task.props.size = size
- }
- if (space.task.props.totalSize) {
- space.task.props.progress = (100 * space.task.props.size) / space.task.props.totalSize
- }
- return true
- }
-
- private async readWatchedPath<T>(rPath: string, publishedPath: string | undefined, read: (rPath: string) => Promise<T>): Promise<T | undefined> {
- if (!publishedPath) return read(rPath)
- if (await isPathExists(rPath)) {
- try {
- const value = await read(rPath)
- // The temporary path may be moved atomically while it is being read.
- if (await isPathExists(rPath)) return value
- } catch (e) {
- // Retry the published path only when the temporary path disappeared during publication.
- if (e?.code !== 'ENOENT') throw e
- }
- }
- if (await isPathExists(publishedPath)) return read(publishedPath)
- }
}
diff --git a/backend/src/applications/files/utils/download-file.spec.ts b/backend/src/applications/files/utils/download-file.spec.ts
index 5da5d430..a92ffb7f 100644
--- a/backend/src/applications/files/utils/download-file.spec.ts
+++ b/backend/src/applications/files/utils/download-file.spec.ts
@@ -247,7 +247,7 @@ describe(DownloadFile.name, () => {
maxSize: 1024
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024)
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)
})
it('rejects missing content-length for space downloads even when maxSize is provided', async () => {
@@ -278,7 +278,21 @@ describe(DownloadFile.name, () => {
})
expect(space.willExceedQuota).toHaveBeenCalledWith(12)
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12)
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)
+ })
+
+ it('forwards the progress callback to the stream writer', async () => {
+ const stream = Readable.from(['abc'])
+ const onProgress = vi.fn()
+ http.axiosRef
+ .mockResolvedValueOnce(response('8.8.8.8', { 'content-length': '12' }))
+ .mockResolvedValueOnce({ ...response('8.8.8.8'), data: stream })
+
+ await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt', {
+ onProgress
+ })
+
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, onProgress)
})
it('allows zero content-length and guards the written stream at zero bytes', async () => {
@@ -289,7 +303,7 @@ describe(DownloadFile.name, () => {
await new DownloadFile(http as unknown as HttpService).download({ url: 'https://example.test/file.txt' }, '/tmp/file.txt')
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0)
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 0, undefined, undefined)
})
it('rejects redirects on GET after the HEAD URL has been resolved', async () => {
@@ -318,7 +332,7 @@ describe(DownloadFile.name, () => {
allowPrivateIP: true
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12)
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 12, undefined, undefined)
expect(http.axiosRef).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
@@ -343,6 +357,6 @@ describe(DownloadFile.name, () => {
maxSize: 1024
})
- expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024)
+ expect(writeFromStream).toHaveBeenCalledWith('/tmp/file.txt', stream, 0, 1024, undefined, undefined)
})
})
diff --git a/backend/src/applications/files/utils/download-file.ts b/backend/src/applications/files/utils/download-file.ts
index 5a2e53c2..6e05e8da 100644
--- a/backend/src/applications/files/utils/download-file.ts
+++ b/backend/src/applications/files/utils/download-file.ts
@@ -9,7 +9,6 @@ import { AxiosHeaders, type AxiosRequestConfig, type AxiosResponse } from 'axios
import ipaddr from 'ipaddr.js'
import { HTTP_METHOD } from '../../applications.constants'
import type { SpaceEnv } from '../../spaces/models/space-env.model'
-import { FILE_OPERATION } from '../constants/operations'
import type { DownloadFileDto } from '../dto/file-operations.dto'
import { FileTaskEvent } from '../events/file-events'
import type { DownloadFileContentInfo, DownloadFileOptions } from '../interfaces/download-file.interface'
@@ -63,7 +62,7 @@ export class DownloadFile {
if (maxSize === null || (contentLength === null && options?.space)) {
throw new FileError(HttpStatus.BAD_REQUEST, FILE_ERROR_MESSAGES.DOWNLOAD_INVALID_CONTENT_LENGTH)
}
- if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath, dstPath)
+ if (contentLength !== null) this.prepareSpace(options?.space, contentLength, options?.publishedPath || dstPath)
// The HEAD request resolved redirects; the GET must target that final URL directly.
const { response: getRes } = await this.request(
@@ -71,11 +70,7 @@ export class DownloadFile {
{ method: HTTP_METHOD.GET, responseType: 'stream', signal: options?.signal, ...identityEncodingConfig },
{ allowPrivateIP: options?.allowPrivateIP, maxRedirects: 0 }
)
- if (options?.signal) {
- await writeFromStream(dstPath, getRes.data, 0, maxSize, options.signal)
- } else {
- await writeFromStream(dstPath, getRes.data, 0, maxSize)
- }
+ await writeFromStream(dstPath, getRes.data, 0, maxSize, options?.signal, options?.onProgress)
}
private safeLookup(hostname: string, options: Parameters<LookupFunction>[1], cb: Parameters<LookupFunction>[2]): void {
@@ -173,18 +168,14 @@ export class DownloadFile {
return Number.isSafeInteger(contentLength) ? contentLength : null
}
- private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string, dstPath: string): void {
+ private prepareSpace(space: SpaceEnv | undefined, contentLength: number, publishedPath: string): void {
if (!space) return
if (space.willExceedQuota(contentLength)) {
throw new FileError(HttpStatus.INSUFFICIENT_STORAGE, FILE_ERROR_MESSAGES.STORAGE_QUOTA_EXCEEDED)
}
if (space.task?.cacheKey) {
- space.task.props.totalSize = contentLength
- if (publishedPath === dstPath) {
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, dstPath)
- } else {
- FileTaskEvent.emit('startWatch', space, FILE_OPERATION.DOWNLOAD, publishedPath, dstPath)
- }
+ space.task.props = { ...space.task.props, progress: 1, size: 0, totalSize: contentLength }
+ FileTaskEvent.emit('startWatch', space, publishedPath)
}
}
}
diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.ts
index 7d78dfff..ec668777 100644
--- a/backend/src/applications/files/utils/files.spec.ts
+++ b/backend/src/applications/files/utils/files.spec.ts
@@ -1,11 +1,12 @@
import { HttpStatus } from '@nestjs/common'
-import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import fs, { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { Readable } from 'node:stream'
+import fse from 'fs-extra'
import { FileError } from '../models/file-error'
import { FILE_ERROR_MESSAGES } from './errors'
-import { isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'
+import { isCrossDevice, isPathInside, makeTempDir, tempFilePath, writeFromStream } from './files'
describe(isPathInside.name, () => {
const basePath = path.join(path.sep, 'tmp', 'output')
@@ -28,6 +29,23 @@ describe(isPathInside.name, () => {
)
})
+describe(isCrossDevice.name, () => {
+ afterEach(() => {
+ vi.restoreAllMocks()
+ })
+
+ it('compares the source device with the nearest existing destination parent', async () => {
+ vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)
+ const statSpy = vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
+ vi.spyOn(fse, 'pathExists').mockResolvedValueOnce(false).mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ const dstPath = path.join(path.sep, 'missing', 'parent', 'destination.txt')
+
+ await expect(isCrossDevice('/source.txt', dstPath)).resolves.toBe(true)
+
+ expect(statSpy).toHaveBeenCalledWith(path.parse(dstPath).root)
+ })
+})
+
describe(writeFromStream.name, () => {
let tmpDir: string
@@ -75,6 +93,17 @@ describe(writeFromStream.name, () => {
name: 'AbortError'
})
})
+
+ it('reports bytes written through the progress callback', async () => {
+ const filePath = path.join(tmpDir, 'progress.txt')
+ const onProgress = vi.fn()
+
+ await writeFromStream(filePath, Readable.from([Buffer.from('abc'), Buffer.from('de')]), 0, 5, undefined, onProgress)
+
+ expect(onProgress).toHaveBeenCalledTimes(2)
+ expect(onProgress).toHaveBeenNthCalledWith(1, 3)
+ expect(onProgress).toHaveBeenNthCalledWith(2, 2)
+ })
})
describe(makeTempDir.name, () => {
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index a3841ec0..10ce54a1 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -52,6 +52,21 @@ export function isPathExists(rPath: string): Promise<boolean> {
return fse.pathExists(rPath)
}
+async function existingParentPath(rPath: string): Promise<string> {
+ let parentPath = path.dirname(rPath)
+ while (!(await isPathExists(parentPath))) {
+ const nextParentPath = path.dirname(parentPath)
+ if (nextParentPath === parentPath) break
+ parentPath = nextParentPath
+ }
+ return parentPath
+}
+
+export async function isCrossDevice(srcPath: string, dstPath: string): Promise<boolean> {
+ const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])
+ return srcStats.dev !== dstParentStats.dev
+}
+
export async function isPathIsReadable(rPath: string): Promise<boolean> {
try {
await fs.access(rPath, fs.constants.R_OK)
@@ -199,23 +214,40 @@ export async function checksumFile(filePath: string, alg: string): Promise<strin
return hash.digest('hex')
}
-export function writeFromStream(rPath: string, stream: Readable, start: number = 0, maxSize?: number, signal?: AbortSignal): Promise<void> {
+export function createProgressTransform(onProgress: (bytes: number) => void): Transform {
+ return new Transform({
+ transform(chunk: Buffer, _encoding, callback) {
+ onProgress(chunk.length)
+ callback(null, chunk)
+ }
+ })
+}
+
+export function writeFromStream(
+ rPath: string,
+ stream: Readable,
+ start: number = 0,
+ maxSize?: number,
+ signal?: AbortSignal,
+ onProgress?: (bytes: number) => void
+): Promise<void> {
const dst: WriteStream = createWriteStream(rPath, { flags: start ? 'a' : 'w', start: start, highWaterMark: DEFAULT_HIGH_WATER_MARK })
- if (maxSize === undefined) {
+ if (maxSize === undefined && !onProgress) {
return pipeline(stream, dst, { signal })
}
let received = start
- const limitSize = new Transform({
+ const progress = new Transform({
transform(chunk, _encoding, callback) {
received += chunk.length
- if (received > maxSize) {
+ if (maxSize !== undefined && received > maxSize) {
callback(maxFileSizeExceededError())
return
}
+ onProgress?.(chunk.length)
callback(null, chunk)
}
})
- return pipeline(stream, limitSize, dst, { signal })
+ return pipeline(stream, progress, dst, { signal })
}
export async function writeFromStreamAndChecksum(rPath: string, stream: Readable, hasRange: number, alg: string): Promise<string> {
diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.ts
index e159c448..d1cc10a4 100644
--- a/backend/src/applications/files/utils/tasks.ts
+++ b/backend/src/applications/files/utils/tasks.ts
@@ -2,7 +2,7 @@ import { Dirent } from 'node:fs'
import fs from 'node:fs/promises'
import path from 'node:path'
import { FILE_OPERATION } from '../constants/operations'
-import { isPathExists, walkDir } from './files'
+import { isCrossDevice, walkDir } from './files'
import type { FileTaskProps } from '../models/file-task'
export function taskTemporaryPrefix(cacheKey: string): string {
@@ -20,21 +20,6 @@ export async function createTaskTemporaryDir(parentPath: string, cacheKey: strin
return temporaryPath
}
-async function existingParentPath(dstPath: string): Promise<string> {
- let parentPath = path.dirname(dstPath)
- while (!(await isPathExists(parentPath))) {
- const nextParentPath = path.dirname(parentPath)
- if (nextParentPath === parentPath) break
- parentPath = nextParentPath
- }
- return parentPath
-}
-
-export async function isCrossDeviceMove(srcPath: string, dstPath: string): Promise<boolean> {
- const [srcStats, dstParentStats] = await Promise.all([fs.lstat(srcPath), existingParentPath(dstPath).then((parentPath) => fs.stat(parentPath))])
- return srcStats.dev !== dstParentStats.dev
-}
-
export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dstPath?: string): Promise<boolean> {
switch (type) {
case FILE_OPERATION.COPY:
@@ -46,7 +31,7 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, d
case FILE_OPERATION.DELETE:
if (!dstPath) return false
try {
- return await isCrossDeviceMove(srcPath, dstPath)
+ return await isCrossDevice(srcPath, dstPath)
} catch {
return false
}
diff --git a/backend/src/applications/files/utils/untar-file.spec.ts b/backend/src/applications/files/utils/untar-file.spec.ts
index 052b6d8c..e7eb1b23 100644
--- a/backend/src/applications/files/utils/untar-file.spec.ts
+++ b/backend/src/applications/files/utils/untar-file.spec.ts
@@ -3,7 +3,7 @@ import { access, mkdir, mkdtemp, readlink, rm, symlink, writeFile } from 'node:f
import os from 'node:os'
import path from 'node:path'
import { create } from 'tar'
-import { checkTarEntry, extractTar } from './untar-file'
+import { checkTarEntry, extractTar, isTarDirectory } from './untar-file'
describe(extractTar.name, () => {
const baseOutputDir = path.join(path.sep, 'tmp', 'output')
@@ -20,6 +20,12 @@ describe(extractTar.name, () => {
)
})
+ it('recognizes regular and GNU dump directories', () => {
+ expect(isTarDirectory('Directory')).toBe(true)
+ expect(isTarDirectory('GNUDumpDir')).toBe(true)
+ expect(isTarDirectory('File')).toBe(false)
+ })
+
let tmpDir: string
let sourceDir: string
let outputDir: string
@@ -40,12 +46,26 @@ describe(extractTar.name, () => {
})
it('extracts symbolic links targeting the output directory', async () => {
+ const onEntry = vi.fn()
await symlink('./v2', path.join(sourceDir, 'docs', 'latest'))
await createArchive()
- await extractTar(archivePath, outputDir, false)
+ await extractTar(archivePath, outputDir, false, undefined, undefined, onEntry)
await expect(readlink(path.join(outputDir, 'docs', 'latest'))).resolves.toBe('./v2')
+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/', isDirectory: true, size: 0 })
+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/latest', isDirectory: false, size: 0 })
+ })
+
+ it('reports decompressed file bytes through the entry transform', async () => {
+ const onEntry = vi.fn()
+ await writeFile(path.join(sourceDir, 'docs', 'file.txt'), 'abc')
+ await createArchive()
+
+ await extractTar(archivePath, outputDir, false, undefined, undefined, onEntry)
+
+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/file.txt', isDirectory: false, size: 0 })
+ expect(onEntry).toHaveBeenCalledWith({ path: 'docs/file.txt', isDirectory: false, size: 3 })
})
it('rejects symbolic links escaping the output directory', async () => {
diff --git a/backend/src/applications/files/utils/untar-file.ts b/backend/src/applications/files/utils/untar-file.ts
index e824fe24..411fe147 100644
--- a/backend/src/applications/files/utils/untar-file.ts
+++ b/backend/src/applications/files/utils/untar-file.ts
@@ -2,8 +2,9 @@ import { createReadStream } from 'node:fs'
import path from 'node:path'
import { pipeline } from 'node:stream/promises'
import { extract, type ReadEntry, type Unpack } from 'tar'
+import type { FileTaskExtractionEntry } from '../interfaces/file-task.interface'
import { storageQuotaExceededError } from './errors'
-import { isPathInside } from './files'
+import { createProgressTransform, isPathInside } from './files'
export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' | 'path' | 'linkpath'>): boolean {
if (entry.type === 'Link') {
@@ -21,7 +22,18 @@ export function checkTarEntry(outputDir: string, entry: Pick<ReadEntry, 'type' |
return true
}
-export async function extractTar(filePath: string, outputDir: string, gzip: boolean, maxExtractedSize?: number, signal?: AbortSignal): Promise<void> {
+export function isTarDirectory(type: ReadEntry['type']): boolean {
+ return type === 'Directory' || type === 'GNUDumpDir'
+}
+
+export async function extractTar(
+ filePath: string,
+ outputDir: string,
+ gzip: boolean,
+ maxExtractedSize?: number,
+ signal?: AbortSignal,
+ onEntry?: (entry: FileTaskExtractionEntry) => void
+): Promise<void> {
let validationError: Error | undefined
let extractedSize = 0
const srcStream = createReadStream(filePath)
@@ -40,6 +52,15 @@ export async function extractTar(filePath: string, outputDir: string, gzip: bool
preserveOwner: false,
preservePaths: false,
strict: true,
+ transform: onEntry
+ ? (entry) => {
+ let extractedEntrySize = 0
+ return createProgressTransform((bytes) => {
+ extractedEntrySize += bytes
+ onEntry({ path: entry.path, isDirectory: false, size: extractedEntrySize })
+ }) as unknown as ReadEntry
+ }
+ : undefined,
filter: (_path, entry) => {
if (validationError) return false
try {
@@ -48,7 +69,10 @@ export async function extractTar(filePath: string, outputDir: string, gzip: bool
if (maxExtractedSize !== undefined && extractedSize > maxExtractedSize) {
throw storageQuotaExceededError()
}
- return !('type' in entry) || checkTarEntry(outputDir, entry)
+ if (!('type' in entry)) return true
+ if (!checkTarEntry(outputDir, entry)) return false
+ onEntry?.({ path: entry.path, isDirectory: isTarDirectory(entry.type), size: 0 })
+ return true
} catch (e) {
return abortExtraction(e as Error)
}
diff --git a/backend/src/applications/files/utils/unzip-file.spec.ts b/backend/src/applications/files/utils/unzip-file.spec.ts
index 3c401e55..2b0eeb90 100644
--- a/backend/src/applications/files/utils/unzip-file.spec.ts
+++ b/backend/src/applications/files/utils/unzip-file.spec.ts
@@ -58,10 +58,13 @@ describe(extractZip.name, () => {
it('extracts entries inside the output directory', async () => {
const extractedPath = path.join(outputDir, 'safe.txt')
- await writeFile(archivePath, createZip('safe.txt'))
+ const onEntry = vi.fn()
+ await writeFile(archivePath, createZip('safe.txt', Buffer.from('abc')))
- await expect(extractZip(archivePath, outputDir)).resolves.toBeUndefined()
+ await expect(extractZip(archivePath, outputDir, undefined, undefined, onEntry)).resolves.toBeUndefined()
await expect(access(extractedPath)).resolves.toBeUndefined()
+ expect(onEntry).toHaveBeenCalledWith({ path: 'safe.txt', isDirectory: false, size: 0 })
+ expect(onEntry).toHaveBeenCalledWith({ path: 'safe.txt', isDirectory: false, size: 3 })
})
it('rejects entries outside the output directory', async () => {
diff --git a/backend/src/applications/files/utils/unzip-file.ts b/backend/src/applications/files/utils/unzip-file.ts
index 574c11a7..9ac41ffa 100644
--- a/backend/src/applications/files/utils/unzip-file.ts
+++ b/backend/src/applications/files/utils/unzip-file.ts
@@ -4,12 +4,19 @@ import { pipeline } from 'node:stream/promises'
import { promisify } from 'node:util'
import { Entry, open as openZip, Options, ZipFile } from 'yauzl'
import { DEFAULT_HIGH_WATER_MARK } from '../constants/files'
+import type { FileTaskExtractionEntry } from '../interfaces/file-task.interface'
import { storageQuotaExceededError } from './errors'
-import { isPathInside, makeDir } from './files'
+import { createProgressTransform, isPathInside, makeDir } from './files'
const openZipAsync: (path: string, options: Options) => Promise<ZipFile> = promisify(openZip)
-export async function extractZip(filePath: string, outputDir: string, maxExtractedSize?: number, signal?: AbortSignal): Promise<void> {
+export async function extractZip(
+ filePath: string,
+ outputDir: string,
+ maxExtractedSize?: number,
+ signal?: AbortSignal,
+ onEntry?: (entry: FileTaskExtractionEntry) => void
+): Promise<void> {
// Reject entries whose actual decompressed size differs from their metadata.
const zipFile = await openZipAsync(filePath, { lazyEntries: true, validateEntrySizes: true })
const openReadStream = promisify(zipFile.openReadStream.bind(zipFile))
@@ -44,12 +51,28 @@ export async function extractZip(filePath: string, outputDir: string, maxExtract
}
if (isDir) {
await makeDir(fullPath, true)
+ onEntry?.({ path: entry.fileName, isDirectory: true, size: 0 })
zipFile.readEntry()
} else {
// make sure parent exists
await makeDir(path.dirname(fullPath), true)
const readStream = await openReadStream(entry)
- await pipeline(readStream, fs.createWriteStream(fullPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK }), { signal })
+ const writeStream = fs.createWriteStream(fullPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ if (onEntry) {
+ let extractedEntrySize = 0
+ onEntry({ path: entry.fileName, isDirectory: false, size: 0 })
+ await pipeline(
+ readStream,
+ createProgressTransform((bytes) => {
+ extractedEntrySize += bytes
+ onEntry({ path: entry.fileName, isDirectory: false, size: extractedEntrySize })
+ }),
+ writeStream,
+ { signal }
+ )
+ } else {
+ await pipeline(readStream, writeStream, { signal })
+ }
zipFile.readEntry()
}
} catch (err) {