Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): handle EXDEV fallbacks for task moves

johaven0b0d10a943a6
diff --git a/backend/src/applications/files/interfaces/file-task.interface.ts b/backend/src/applications/files/interfaces/file-task.interface.ts
index 1091dfb3..a5917e7b 100644
--- a/backend/src/applications/files/interfaces/file-task.interface.ts
+++ b/backend/src/applications/files/interfaces/file-task.interface.ts
@@ -8,7 +8,7 @@ export interface FileTasksPollResponse {
}
export interface FileTaskTransferOptions {
beforeCommit?: () => Promise<void>
- crossDevice?: boolean
+ beforeTransfer?: () => Promise<void>
executionId: string
onTransferStart?: () => void
onProgress?: (bytes: number) => void
diff --git a/backend/src/applications/files/models/file-task.ts b/backend/src/applications/files/models/file-task.ts
index 97314df4..fe7a7cff 100644
--- a/backend/src/applications/files/models/file-task.ts
+++ b/backend/src/applications/files/models/file-task.ts
@@ -36,9 +36,14 @@ export class FileTask {
this.cancellable = cancellable
this.path = path
this.name = name
- if (type === FILE_OPERATION.COPY || type === FILE_OPERATION.MOVE || type === FILE_OPERATION.DOWNLOAD) {
+ if (
+ type === FILE_OPERATION.COPY ||
+ type === FILE_OPERATION.MOVE ||
+ type === FILE_OPERATION.DOWNLOAD ||
+ (type === FILE_OPERATION.DELETE && cancellable)
+ ) {
this.props = { progress: 1 }
- if (type !== FILE_OPERATION.DOWNLOAD) {
+ if (type === FILE_OPERATION.COPY || type === FILE_OPERATION.MOVE) {
// copy move operation
this.props.src = { name: name, path: path }
}
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
index 1b7b3d35..80975398 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.spec.ts
@@ -391,15 +391,37 @@ describe(FilesTasksManager.name, () => {
it('should run a non-cancellable task without an abort signal', async () => {
filesMethods.doWork.mockResolvedValueOnce(undefined)
const user = { id: 7 } as any
- const space = { realPath: '/data/users/john/files/document.txt', url: 'files/personal/document.txt' } as any
- const dto = { dstDirectory: 'files/personal' }
+ const space = {
+ inTrashRepository: true,
+ realPath: '/data/users/john/trash/document.txt',
+ url: 'trash/personal/document.txt'
+ } as any
- const task = await filesTasksManager.createTask(FILE_OPERATION.MOVE, user, space, dto, 'doWork')
+ const task = await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')
await flushPromises()
- expect(spacesManager.spaceEnv).toHaveBeenCalledWith(user, ['files', 'personal', 'document.txt'])
expect(task.cancellable).toBe(false)
- expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, dto)
+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, null)
+ })
+
+ it('should stream trash deletes without relying on storage-specific ids', async () => {
+ filesMethods.doWork.mockReturnValueOnce(new Promise<void>(() => undefined))
+ const user = { id: 7 } as any
+ const space = {
+ dbFile: { path: 'documents/report.png', shareExternalId: 33 },
+ inFilesRepository: false,
+ inSharesRepository: true,
+ inTrashRepository: false,
+ realPath: '/mnt/archive/documents/report.png',
+ root: { externalPath: '/mnt/archive', file: { space: { alias: 'project', id: 11 } } },
+ url: 'shares/project/documents/report.png'
+ } as any
+
+ const task = await filesTasksManager.createTask(FILE_OPERATION.DELETE, user, space, null, 'doWork')
+
+ expect(task.cancellable).toBe(true)
+ expect(task.props).toEqual({ progress: 1 })
+ expect(filesMethods.doWork).toHaveBeenCalledWith(user, space, null, expect.any(AbortSignal))
})
it('should queue tasks when a user already has three running tasks', async () => {
diff --git a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
index bb38ef49..08581917 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-manager.service.ts
@@ -54,9 +54,9 @@ export class FilesTasksManager implements OnModuleDestroy {
async createTask(type: FILE_OPERATION, user: UserModel, space: SpaceEnv, dto: any, method: string): Promise<FileTask> {
const taskId: string = crypto.randomUUID()
const cacheKey = FilesTasksManager.getCacheKey(user.id, taskId)
- // MOVE and DELETE need their effective destination to determine whether they cross devices.
+ // MOVE and DELETE only need an effective destination to support a cancellable streamed fallback.
const dstPath = await this.taskDestinationPath(type, user, space, dto)
- const cancellable = await isTaskCancellable(type, space.realPath, dstPath)
+ const cancellable = isTaskCancellable(type, dstPath)
const newTask = new FileTask(taskId, type, dirName(space.url), fileName(space.url), cancellable)
await this.storeTask(cacheKey, newTask, FileTaskStatus.QUEUED)
space.task = { id: taskId, type, cacheKey, props: {} }
@@ -206,7 +206,7 @@ export class FilesTasksManager implements OnModuleDestroy {
}
private runTask(task: FileTaskQueueItem): void {
- // Only cancellable tasks receive a signal; its absence keeps same-device moves on the atomic path.
+ // Transfer-capable tasks receive a signal; successful native renames still stay atomic.
const controller = this.watchCancellation(task.task, task.user.id, task.cacheKey)
const taskPromise = controller
? this.filesMethods[task.method](task.user, task.space, task.dto, controller.signal)
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
index d50b705b..480a0b29 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.spec.ts
@@ -197,21 +197,68 @@ describe(FilesTasksTransfer.name, () => {
it('copies then removes the source for a cross-device move', async () => {
const signal = new AbortController().signal
await writeFile(srcPath, 'content')
+ mockCrossDevice()
- await moveAbortable({ crossDevice: true, signal, stagingDir })
+ await moveAbortable({ signal, stagingDir })
await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
await expect(readdir(stagingDir)).resolves.toEqual([])
})
+ it('keeps same-mount moves atomic without initializing streamed progress', async () => {
+ const signal = new AbortController().signal
+ const beforeTransfer = vi.fn()
+ await writeFile(srcPath, 'content')
+
+ await moveAbortable({ beforeTransfer, signal, stagingDir })
+
+ expect(beforeTransfer).not.toHaveBeenCalled()
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe('content')
+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ })
+
+ it('falls back to instrumented copies when rename reports EXDEV despite device hints', async () => {
+ const signal = new AbortController().signal
+ const beforeCommit = vi.fn()
+ const beforeTransfer = vi.fn()
+ const content = 'content'
+ const temporaryPath = filesUtils.temporaryFilePath(stagingDir, dstPath, FILE_OPERATION.MOVE, taskId)
+ let transferredBytes = 0
+ await writeFile(srcPath, content)
+ const renameSpy = vi
+ .spyOn(fs, 'rename')
+ .mockRejectedValueOnce(Object.assign(new Error('cross-mount move'), { code: 'EXDEV' }))
+ .mockRejectedValueOnce(Object.assign(new Error('cross-mount staging'), { code: 'EXDEV' }))
+
+ await moveAbortable({
+ beforeCommit,
+ beforeTransfer,
+ onProgress: (bytes: number) => {
+ transferredBytes += bytes
+ },
+ signal,
+ stagingDir
+ })
+
+ expect(beforeCommit).toHaveBeenCalledOnce()
+ expect(beforeTransfer).toHaveBeenCalledOnce()
+ expect(transferredBytes).toBe(Buffer.byteLength(content))
+ expect(renameSpy).toHaveBeenNthCalledWith(1, srcPath, dstPath)
+ expect(renameSpy).toHaveBeenNthCalledWith(2, temporaryPath, dstPath)
+ await expect(readFile(dstPath, 'utf8')).resolves.toBe(content)
+ await expect(access(srcPath)).rejects.toMatchObject({ code: 'ENOENT' })
+ await expect(readdir(stagingDir)).resolves.toEqual([])
+ })
+
it('reports source cleanup failure after keeping the published destination', async () => {
const signal = new AbortController().signal
const error = new Error('source cleanup failed')
await writeFile(srcPath, 'content')
+ mockCrossDevice()
vi.spyOn(filesUtils, 'removeFiles').mockRejectedValueOnce(error)
- const cleanupError = await moveAbortable({ crossDevice: true, signal, stagingDir })
+ const cleanupError = await moveAbortable({ signal, stagingDir })
expect(cleanupError).toBeInstanceOf(SourceCleanupError)
expect(cleanupError).toMatchObject({
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
index af9fee53..20bbdcd1 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
@@ -16,11 +16,10 @@ import {
isCrossDevice,
isInternalTemporaryEntry,
isPathExists,
- moveFiles,
removeFiles,
temporaryFilePath
} from '../../utils/files'
-import { countDirEntriesAndSize } from '../../utils/tasks'
+import { countDirEntriesAndSize, isCrossDeviceError } from '../../utils/tasks'
import { SourceCleanupError } from '../../models/file-error'
@Injectable()
@@ -58,11 +57,10 @@ export class FilesTasksTransfer {
signal: AbortSignal,
deleteDestination: () => Promise<void>
): Promise<SourceCleanupError | undefined> {
- await this.initializeTaskProps(srcSpace, isDir)
const beforeCommit = this.prepareTaskDestination(srcSpace, dstSpace, overwrite, deleteDestination)
return this.moveAbortable(srcSpace.realPath, dstSpace.realPath, {
beforeCommit,
- crossDevice: true,
+ beforeTransfer: () => this.initializeTaskProps(srcSpace, isDir),
executionId: srcSpace.task!.id,
onProgress: this.createByteProgressHandler(srcSpace),
onTransferStart: () => this.startTransferTaskWatch(srcSpace, dstSpace.realPath),
@@ -81,23 +79,15 @@ export class FilesTasksTransfer {
signal: AbortSignal | undefined,
prepareDestination: () => Promise<void>
): Promise<SourceCleanupError | undefined> {
- await this.initializeTaskProps(space, isDir)
- if (!signal) {
- // Moving to trash on the same device stays atomic and does not copy bytes.
- this.startTransferTaskWatch(space, trashFile)
- await prepareDestination()
- await moveFiles(space.realPath, trashFile, true)
- return
- }
return this.moveAbortable(space.realPath, trashFile, {
beforeCommit: prepareDestination,
- crossDevice: true,
+ beforeTransfer: () => this.initializeTaskProps(space, isDir),
executionId: space.task!.id,
onProgress: this.createByteProgressHandler(space),
onTransferStart: () => this.startTransferTaskWatch(space, trashFile),
operation: FILE_OPERATION.DELETE,
overwrite: true,
- signal,
+ signal: signal ?? new AbortController().signal,
stagingDir
})
}
@@ -192,7 +182,7 @@ export class FilesTasksTransfer {
await this.copyEntry(srcPath, dstPath, recursive, preserveTimestamps, signal, onProgress)
} catch (e) {
if (transferStarted) {
- await this.cleanupAfterFailure(dstPath)
+ await this.removeBestEffort(dstPath)
}
throw e
}
@@ -205,24 +195,30 @@ export class FilesTasksTransfer {
signal.throwIfAborted()
await beforeCommit?.()
await this.prepareDestination(dstPath, overwrite)
- await fs.rename(temporaryPath, dstPath)
+ await this.publishTemporaryEntry(temporaryPath, dstPath, recursive, preserveTimestamps, signal)
} catch (e) {
- await this.cleanupAfterFailure(temporaryPath)
+ await this.removeBestEffort(temporaryPath)
throw e
}
}
private async moveAbortable(srcPath: string, dstPath: string, options: FileTaskTransferOptions): Promise<SourceCleanupError | undefined> {
- const { beforeCommit, executionId, onProgress, onTransferStart, operation, overwrite = false, signal, stagingDir } = options
- const crossDevice = options.crossDevice ?? (await isCrossDevice(srcPath, dstPath))
+ const { beforeCommit, beforeTransfer, executionId, onProgress, onTransferStart, operation, overwrite = false, signal, stagingDir } = options
+ const crossDevice = await isCrossDevice(srcPath, dstPath)
+ let streamedBeforeCommit = beforeCommit
if (!crossDevice) {
signal.throwIfAborted()
await beforeCommit?.()
- await moveFiles(srcPath, dstPath, overwrite)
- return
+ streamedBeforeCommit = undefined
+ if (!overwrite && srcPath.toLowerCase() !== dstPath.toLowerCase() && (await isPathExists(dstPath))) {
+ throw this.destinationExistsError()
+ }
+ if (await this.tryRename(srcPath, dstPath)) return
}
+ signal.throwIfAborted()
+ await beforeTransfer?.()
await this.copyAbortable(srcPath, dstPath, {
- beforeCommit,
+ beforeCommit: streamedBeforeCommit,
executionId,
onProgress,
onTransferStart,
@@ -238,6 +234,34 @@ export class FilesTasksTransfer {
}
}
+ private async publishTemporaryEntry(
+ temporaryPath: string,
+ dstPath: string,
+ recursive: boolean,
+ preserveTimestamps: boolean,
+ signal: AbortSignal
+ ): Promise<void> {
+ if (await this.tryRename(temporaryPath, dstPath)) return
+ try {
+ // Bytes were already accounted for while filling the staging entry.
+ await this.copyEntry(temporaryPath, dstPath, recursive, preserveTimestamps, signal)
+ } catch (error) {
+ await this.removeBestEffort(dstPath)
+ throw error
+ }
+ await this.removeBestEffort(temporaryPath)
+ }
+
+ private async tryRename(srcPath: string, dstPath: string): Promise<boolean> {
+ try {
+ await fs.rename(srcPath, dstPath)
+ return true
+ } catch (error) {
+ if (isCrossDeviceError(error)) return false
+ throw error
+ }
+ }
+
private async copyEntry(
srcPath: string,
dstPath: string,
@@ -278,16 +302,20 @@ export class FilesTasksTransfer {
private async prepareDestination(dstPath: string, overwrite: boolean): Promise<void> {
if (!(await isPathExists(dstPath))) return
if (!overwrite) {
- throw Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })
+ throw this.destinationExistsError()
}
await removeFiles(dstPath)
}
- private async cleanupAfterFailure(rPath: string): Promise<void> {
+ private destinationExistsError(): NodeJS.ErrnoException {
+ return Object.assign(new Error('Destination already exists'), { code: 'EEXIST' })
+ }
+
+ private async removeBestEffort(rPath: string): Promise<void> {
try {
await removeFiles(rPath)
} catch {
- // Cleanup is best-effort and must not replace the transfer error.
+ // Cleanup is best-effort and must not replace the transfer result.
}
}
}
diff --git a/backend/src/applications/files/utils/tasks.spec.ts b/backend/src/applications/files/utils/tasks.spec.ts
index ae276981..6fb3423e 100644
--- a/backend/src/applications/files/utils/tasks.spec.ts
+++ b/backend/src/applications/files/utils/tasks.spec.ts
@@ -6,12 +6,10 @@ import { countDirEntriesAndSize, isTaskCancellable } from './tasks'
describe('file task utilities', () => {
let tmpDir: string
- let srcPath: string
let dstPath: string
beforeEach(async () => {
tmpDir = await mkdtemp(path.join(os.tmpdir(), 'file-tasks-'))
- srcPath = path.join(tmpDir, 'source.txt')
dstPath = path.join(tmpDir, 'destination.txt')
})
@@ -20,24 +18,15 @@ describe('file task utilities', () => {
vi.restoreAllMocks()
})
- it('marks intrinsically abortable operations as cancellable', async () => {
- await expect(isTaskCancellable(FILE_OPERATION.COPY, '/source')).resolves.toBe(true)
- await expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD, '/source')).resolves.toBe(true)
+ it('marks intrinsically abortable operations as cancellable', () => {
+ expect(isTaskCancellable(FILE_OPERATION.COPY)).toBe(true)
+ expect(isTaskCancellable(FILE_OPERATION.DOWNLOAD)).toBe(true)
})
- it('requires a cross-device destination for move and delete', async () => {
- await writeFile(srcPath, 'content')
- const lstatSpy = vi.spyOn(fs, 'lstat')
- const statSpy = vi.spyOn(fs, 'stat')
- lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)
- statSpy.mockResolvedValueOnce({ dev: 2 } as any)
-
- await expect(isTaskCancellable(FILE_OPERATION.MOVE, srcPath, dstPath)).resolves.toBe(true)
- lstatSpy.mockResolvedValueOnce({ dev: 1 } as any)
- statSpy.mockResolvedValueOnce({ dev: 2 } as any)
- await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath, dstPath)).resolves.toBe(true)
- await expect(isTaskCancellable(FILE_OPERATION.DELETE, srcPath)).resolves.toBe(false)
- expect(lstatSpy).toHaveBeenCalledWith(srcPath)
+ it('marks moves and trash deletes as cancellable when their destination is known', () => {
+ expect(isTaskCancellable(FILE_OPERATION.MOVE, dstPath)).toBe(true)
+ expect(isTaskCancellable(FILE_OPERATION.DELETE, dstPath)).toBe(true)
+ expect(isTaskCancellable(FILE_OPERATION.DELETE)).toBe(false)
})
it('counts task content while ignoring internal temporary entries', async () => {
diff --git a/backend/src/applications/files/utils/tasks.ts b/backend/src/applications/files/utils/tasks.ts
index 7ad0e760..70d916f7 100644
--- a/backend/src/applications/files/utils/tasks.ts
+++ b/backend/src/applications/files/utils/tasks.ts
@@ -1,10 +1,14 @@
import { Dirent } from 'node:fs'
import fs from 'node:fs/promises'
import { FILE_OPERATION } from '../constants/operations'
-import { isCrossDevice, isInternalTemporaryEntry, walkDir } from './files'
+import { isInternalTemporaryEntry, walkDir } from './files'
import { FileTaskProps, FileTaskStatus } from '../models/file-task'
-export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, dstPath?: string): Promise<boolean> {
+export function isCrossDeviceError(error: unknown): error is NodeJS.ErrnoException {
+ return (error as NodeJS.ErrnoException)?.code === 'EXDEV'
+}
+
+export function isTaskCancellable(type: FILE_OPERATION, dstPath?: string): boolean {
switch (type) {
case FILE_OPERATION.COPY:
case FILE_OPERATION.DOWNLOAD:
@@ -13,12 +17,9 @@ export async function isTaskCancellable(type: FILE_OPERATION, srcPath: string, d
return true
case FILE_OPERATION.MOVE:
case FILE_OPERATION.DELETE:
- if (!dstPath) return false
- try {
- return await isCrossDevice(srcPath, dstPath)
- } catch {
- return false
- }
+ // The real rename capability is only authoritative at execution time.
+ // Supplying a signal keeps the streamed EXDEV fallback cancellable.
+ return Boolean(dstPath)
default:
return false
}