Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): handle EXDEV fallbacks for task moves
johaven0b0d10a943a6
diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.tsindex 1091dfb3..a5917e7b 100644--- a/backend/src/applications/files/interfaces/file-task.interface.ts+++ b/backend/src/applications/files/interfaces/file-task.interface.ts@@ -8,7 +8,7 @@ export interface FileTasksPollResponse {}export interface FileTaskTransferOptions {beforeCommit?: () => Promise<void>- crossDevice?: boolean+ beforeTransfer?: () => Promise<void>executionId: stringonTransferStart?: () => voidonProgress?: (bytes: number) => voiddiff --git a/backend/src/applications/files/models/file-task.ts b/backend/src/applications/files/models/file-task.tsindex 97314df4..fe7a7cff 100644--- a/backend/src/applications/files/models/file-task.ts+++ b/backend/src/applications/files/models/file-task.ts@@ -36,9 +36,14 @@ export class FileTask {this.cancellable = cancellablethis.path = paththis.name = name- if (type === FILE_OPERATION.COPY || type === FILE_OPERATION.MOVE || type === FILE_OPERATION.DOWNLOAD) {+ if (+ type === FILE_OPERATION.COPY ||+ type === FILE_OPERATION.MOVE ||+ type === FILE_OPERATION.DOWNLOAD ||+ (type === FILE_OPERATION.DELETE && cancellable)+ ) {this.props = { progress: 1 }- if (type !== FILE_OPERATION.DOWNLOAD) {+ if (type === FILE_OPERATION.COPY || type === FILE_OPERATION.MOVE) {// copy move operationthis.props.src = { name: name, path: path }}diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.tsindex 1b7b3d35..80975398 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts@@ -391,15 +391,37 @@ describe(FilesTasksManager.name, () => {it('should run a non-cancellable task without an abort signal', async () => {filesMethods.doWork.mockResolvedValueOnce(undefined)const user = { id: 7 } as any- const space = { realPath: '/data/users/john/files/document.txt', url: 'files/personal/document.txt' } as any- const dto = { dstDirectory: 'files/personal' }+ const space = {+ inTrashRepository: true,+ realPath: '/data/users/john/trash/document.txt',+ url: 'trash/personal/document.txt'+ } as any- const task = await filesTasksManager.createTask(FILE_OPERATION.MOVE, user, space, dto, 'doWork')+ const task = await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')await flushPromises()- expect(spacesManager.spaceEnv).toHaveBeenCalledWith(user, ['files', 'personal', 'document.txt'])expect(task.cancellable).toBe(false)- expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, dto)+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, null)+ })++ it('should stream trash deletes without relying on storage-specific ids', async () => {+ filesMethods.doWork.mockReturnValueOnce(new Promise<void>(() => undefined))+ const user = { id: 7 } as any+ const space = {+ dbFile: { path: 'documents/report.png', shareExternalId: 33 },+ inFilesRepository: false,+ inSharesRepository: true,+ inTrashRepository: false,+ realPath: '/mnt/archive/documents/report.png',+ root: { externalPath: '/mnt/archive', file: { space: { alias: 'project', id: 11 } } },+ url: 'shares/project/documents/report.png'+ } as any++ const task = await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')++ expect(task.cancellable).toBe(true)+ expect(task.props).toEqual({ progress: 1 })+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, null, expect.any(AbortSignal))})it('should queue tasks when a user already has three running tasks', async () => {diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.tsindex bb38ef49..08581917 100644--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts@@ -54,9 +54,9 @@ export class FilesTasksManager implements OnModuleDestroy {async createTask(type: FILE_OPERATION, user: UserModel, space: SpaceEnv, dto: any, method: string): Promise<FileTask> {const taskId: string = crypto.randomUUID()const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)- // MOVE and DELETE need their effective destination to determine whether they cross devices.+ // MOVE and DELETE only need an effective destination to support a cancellable streamed fallback.const dstPath = await this.taskDestinationPath(type, user, space, dto)- const cancellable = await isTaskCancellable(type, space.realPath, dstPath)+ const cancellable = isTaskCancellable(type, dstPath)const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url), cancellable)await this.storeTask(cacheKey, newTask, FileTaskStatus.QUEUED)space.task = { id: taskId, type, cacheKey, props: {} }@@ -206,7 +206,7 @@ export class FilesTasksManager implements OnModuleDestroy {}private runTask(task: FileTaskQueueItem): void {- // Only cancellable tasks receive a signal; its absence keeps same-device moves on the atomic path.+ // Transfer-capable tasks receive a signal; successful native renames still stay atomic.const controller = this.watchCancellation(task.task, task.user.id, task.cacheKey)const taskPromise = controller? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.tsindex d50b705b..480a0b29 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts@@ -197,21 +197,68 @@ describe(FilesTasksTransfer.name, () => {it('copies then removes the source for a cross-device move', async () => {const signal = new AbortController().signalawait writeFile(srcPath, 'content')+ mockCrossDevice()- await moveAbortable({ crossDevice: true, signal, stagingDir })+ await moveAbortable({ signal, stagingDir })await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })await expect(readdir(stagingDir)).resolves.toEqual([])})+ it('keeps same-mount moves atomic without initializing streamed progress', async () => {+ const signal = new AbortController().signal+ const beforeTransfer = vi.fn()+ await writeFile(srcPath, 'content')++ await moveAbortable({ beforeTransfer, signal, stagingDir })++ expect(beforeTransfer).not.toHaveBeenCalled()+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })+ })++ it('falls back to instrumented copies when rename reports EXDEV despite device hints', async () => {+ const signal = new AbortController().signal+ const beforeCommit = vi.fn()+ const beforeTransfer = vi.fn()+ const content = 'content'+ const temporaryPath = filesUtils.temporaryFilePath(stagingDir, dstPath, FILE_OPERATION.MOVE, taskId)+ let transferredBytes = 0+ await writeFile(srcPath, content)+ const renameSpy = vi+ .spyOn(fs, 'rename')+ .mockRejectedValueOnce(Object.assign(new Error('cross-mount move'), { code: 'EXDEV' }))+ .mockRejectedValueOnce(Object.assign(new Error('cross-mount staging'), { code: 'EXDEV' }))++ await moveAbortable({+ beforeCommit,+ beforeTransfer,+ onProgress: (bytes: number) => {+ transferredBytes += bytes+ },+ signal,+ stagingDir+ })++ expect(beforeCommit).toHaveBeenCalledOnce()+ expect(beforeTransfer).toHaveBeenCalledOnce()+ expect(transferredBytes).toBe(Buffer.byteLength(content))+ expect(renameSpy).toHaveBeenNthCalledWith(1, srcPath, dstPath)+ expect(renameSpy).toHaveBeenNthCalledWith(2, temporaryPath, dstPath)+ await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })+ await expect(readdir(stagingDir)).resolves.toEqual([])+ })+it('reports source cleanup failure after keeping the published destination', async () => {const signal = new AbortController().signalconst error = new Error('source cleanup failed')await writeFile(srcPath, 'content')+ mockCrossDevice()vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)- const cleanupError = await moveAbortable({ crossDevice: true, signal, stagingDir })+ const cleanupError = await moveAbortable({ signal, stagingDir })expect(cleanupError).toBeInstanceOf(SourceCleanupError)expect(cleanupError).toMatchObject({diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.tsindex af9fee53..20bbdcd1 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts@@ -16,11 +16,10 @@ import {isCrossDevice,isInternalTemporaryEntry,isPathExists,- moveFiles,removeFiles,temporaryFilePath} from '../../utils/files'-import { countDirEntriesAndSize } from '../../utils/tasks'+import { countDirEntriesAndSize, isCrossDeviceError } from '../../utils/tasks'import { SourceCleanupError } from '../../models/file-error'@Injectable()@@ -58,11 +57,10 @@ export class FilesTasksTransfer {signal: AbortSignal,deleteDestination: () => Promise<void>): Promise<SourceCleanupError | undefined> {- await this.initializeTaskProps(srcSpace, isDir)const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {beforeCommit,- crossDevice: true,+ beforeTransfer: () => this.initializeTaskProps(srcSpace, isDir),executionId: srcSpace.task!.id,onProgress: this.createByteProgressHandler(srcSpace),onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),@@ -81,23 +79,15 @@ export class FilesTasksTransfer {signal: AbortSignal | undefined,prepareDestination: () => Promise<void>): Promise<SourceCleanupError | undefined> {- await this.initializeTaskProps(space, isDir)- if (!signal) {- // Moving to trash on the same device stays atomic and does not copy bytes.- this.startTransferTaskWatch(space, trashFile)- await prepareDestination()- await moveFiles(space.realPath, trashFile, true)- return- }return this.moveAbortable(space.realPath, trashFile, {beforeCommit: prepareDestination,- crossDevice: true,+ beforeTransfer: () => this.initializeTaskProps(space, isDir),executionId: space.task!.id,onProgress: this.createByteProgressHandler(space),onTransferStart: () => this.startTransferTaskWatch(space, trashFile),operation: FILE_OPERATION.DELETE,overwrite: true,- signal,+ signal: signal ?? new AbortController().signal,stagingDir})}@@ -192,7 +182,7 @@ export class FilesTasksTransfer {await this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)} catch (e) {if (transferStarted) {- await this.cleanupAfterFailure(dstPath)+ await this.removeBestEffort(dstPath)}throw e}@@ -205,24 +195,30 @@ export class FilesTasksTransfer {signal.throwIfAborted()await beforeCommit?.()await this.prepareDestination(dstPath, overwrite)- await fs.rename(temporaryPath, dstPath)+ await this.publishTemporaryEntry(temporaryPath, dstPath, recursive, preserveTimestamps, signal)} catch (e) {- await this.cleanupAfterFailure(temporaryPath)+ await this.removeBestEffort(temporaryPath)throw e}}private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {- const { beforeCommit, executionId, onProgress, onTransferStart, operation, overwrite = false, signal, stagingDir } = options- const crossDevice = options.crossDevice ?? (await isCrossDevice(srcPath, dstPath))+ const { beforeCommit, beforeTransfer, executionId, onProgress, onTransferStart, operation, overwrite = false, signal, stagingDir } = options+ const crossDevice = await isCrossDevice(srcPath, dstPath)+ let streamedBeforeCommit = beforeCommitif (!crossDevice) {signal.throwIfAborted()await beforeCommit?.()- await moveFiles(srcPath, dstPath, overwrite)- return+ streamedBeforeCommit = undefined+ if (!overwrite && srcPath.toLowerCase() !== dstPath.toLowerCase() && (await isPathExists(dstPath))) {+ throw this.destinationExistsError()+ }+ if (await this.tryRename(srcPath, dstPath)) return}+ signal.throwIfAborted()+ await beforeTransfer?.()await this.copyAbortable(srcPath, dstPath, {- beforeCommit,+ beforeCommit: streamedBeforeCommit,executionId,onProgress,onTransferStart,@@ -238,6 +234,34 @@ export class FilesTasksTransfer {}}+ private async publishTemporaryEntry(+ temporaryPath: string,+ dstPath: string,+ recursive: boolean,+ preserveTimestamps: boolean,+ signal: AbortSignal+ ): Promise<void> {+ if (await this.tryRename(temporaryPath, dstPath)) return+ try {+ // Bytes were already accounted for while filling the staging entry.+ await this.copyEntry(temporaryPath, dstPath, recursive, preserveTimestamps, signal)+ } catch (error) {+ await this.removeBestEffort(dstPath)+ throw error+ }+ await this.removeBestEffort(temporaryPath)+ }++ private async tryRename(srcPath: string, dstPath: string): Promise<boolean> {+ try {+ await fs.rename(srcPath, dstPath)+ return true+ } catch (error) {+ if (isCrossDeviceError(error)) return false+ throw error+ }+ }+private async copyEntry(srcPath: string,dstPath: string,@@ -278,16 +302,20 @@ export class FilesTasksTransfer {private async prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {if (!(await isPathExists(dstPath))) returnif (!overwrite) {- throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })+ throw this.destinationExistsError()}await removeFiles(dstPath)}- private async cleanupAfterFailure(rPath: string): Promise<void> {+ private destinationExistsError(): NodeJS.ErrnoException {+ return Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })+ }++ private async removeBestEffort(rPath: string): Promise<void> {try {await removeFiles(rPath)} catch {- // Cleanup is best-effort and must not replace the transfer error.+ // Cleanup is best-effort and must not replace the transfer result.}}}diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.tsindex ae276981..6fb3423e 100644--- a/backend/src/applications/files/utils/tasks.spec.ts+++ b/backend/src/applications/files/utils/tasks.spec.ts@@ -6,12 +6,10 @@ import { countDirEntriesAndSize, isTaskCancellable } from './tasks'describe('file task utilities', () => {let tmpDir: string- let srcPath: stringlet dstPath: stringbeforeEach(async () => {tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))- srcPath = path.join(tmpDir, 'source.txt')dstPath = path.join(tmpDir, 'destination.txt')})@@ -20,24 +18,15 @@ describe('file task utilities', () => {vi.restoreAllMocks()})- it('marks intrinsically abortable operations as cancellable', async () => {- await expect(isTaskCancellable(FILE_OPERATION.COPY, '/source')).resolves.toBe(true)- await expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD, '/source')).resolves.toBe(true)+ it('marks intrinsically abortable operations as cancellable', () => {+ expect(isTaskCancellable(FILE_OPERATION.COPY)).toBe(true)+ expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD)).toBe(true)})- it('requires a cross-device destination for move and delete', async () => {- await writeFile(srcPath, 'content')- const lstatSpy = vi.spyOn(fs, 'lstat')- const statSpy = vi.spyOn(fs, 'stat')- lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)- statSpy.mockResolvedValueOnce({ dev: 2 } as any)-- await expect(isTaskCancellable(FILE_OPERATION.MOVE, srcPath, dstPath)).resolves.toBe(true)- lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)- statSpy.mockResolvedValueOnce({ dev: 2 } as any)- await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath, dstPath)).resolves.toBe(true)- await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath)).resolves.toBe(false)- expect(lstatSpy).toHaveBeenCalledWith(srcPath)+ it('marks moves and trash deletes as cancellable when their destination is known', () => {+ expect(isTaskCancellable(FILE_OPERATION.MOVE, dstPath)).toBe(true)+ expect(isTaskCancellable(FILE_OPERATION.DELETE, dstPath)).toBe(true)+ expect(isTaskCancellable(FILE_OPERATION.DELETE)).toBe(false)})it('counts task content while ignoring internal temporary entries', async () => {diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.tsindex 7ad0e760..70d916f7 100644--- a/backend/src/applications/files/utils/tasks.ts+++ b/backend/src/applications/files/utils/tasks.ts@@ -1,10 +1,14 @@import { Dirent } from 'node:fs'import fs from 'node:fs/promises'import { FILE_OPERATION } from '../constants/operations'-import { isCrossDevice, isInternalTemporaryEntry, walkDir } from './files'+import { isInternalTemporaryEntry, walkDir } from './files'import { FileTaskProps, FileTaskStatus } from '../models/file-task'-export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dstPath?: string): Promise<boolean> {+export function isCrossDeviceError(error: unknown): error is NodeJS.ErrnoException {+ return (error as NodeJS.ErrnoException)?.code === 'EXDEV'+}++export function isTaskCancellable(type: FILE_OPERATION, dstPath?: string): boolean {switch (type) {case FILE_OPERATION.COPY:case FILE_OPERATION.DOWNLOAD:@@ -13,12 +17,9 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dreturn truecase FILE_OPERATION.MOVE:case FILE_OPERATION.DELETE:- if (!dstPath) return false- try {- return await isCrossDevice(srcPath, dstPath)- } catch {- return false- }+ // The real rename capability is only authoritative at execution time.+ // Supplying a signal keeps the streamed EXDEV fallback cancellable.+ return Boolean(dstPath)default:return false}