Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

perf(backend:files): use reflinks for copy operations

johavend0b4c2c8008b
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
index 20bbdcd1..85aa0166 100644
--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts
@@ -17,7 +17,8 @@ import {
isInternalTemporaryEntry,
isPathExists,
removeFiles,
- temporaryFilePath
+ temporaryFilePath,
+ tryReflink
} from '../../utils/files'
import { countDirEntriesAndSize, isCrossDeviceError } from '../../utils/tasks'
import { SourceCleanupError } from '../../models/file-error'
@@ -283,12 +284,18 @@ export class FilesTasksTransfer {
} else if (stats.isSymbolicLink()) {
await fs.symlink(await fs.readlink(srcPath), dstPath)
} else {
- const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
- const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })
- if (onProgress) {
- await pipeline(src, createProgressTransform(onProgress), dst, { signal })
+ const reflinked = await tryReflink(srcPath, dstPath)
+ if (reflinked) {
+ signal.throwIfAborted()
+ onProgress?.(stats.size)
} else {
- await pipeline(src, dst, { signal })
+ const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })
+ if (onProgress) {
+ await pipeline(src, createProgressTransform(onProgress), dst, { signal })
+ } else {
+ await pipeline(src, dst, { signal })
+ }
}
}
if (!stats.isSymbolicLink()) {
diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.ts
index 8fed7238..8b39322e 100644
--- a/backend/src/applications/files/utils/files.spec.ts
+++ b/backend/src/applications/files/utils/files.spec.ts
@@ -3,8 +3,6 @@ import fs, { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { Readable } from 'node:stream'
-import fse from 'fs-extra'
-import type { MockInstance } from 'vitest'
import { FileError } from '../models/file-error'
import { storageQuotaExceededError } from './errors'
import {
@@ -92,8 +90,7 @@ describe(isCrossDevice.name, () => {
it('compares the source device with the nearest existing destination parent', async () => {
vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)
const statSpy = vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)
- const pathExistsSpy = vi.spyOn(fse, 'pathExists') as unknown as MockInstance<(path: string) => Promise<boolean>>
- pathExistsSpy.mockResolvedValueOnce(false).mockResolvedValueOnce(false).mockResolvedValueOnce(true)
+ vi.spyOn(fs, 'access').mockRejectedValueOnce(new Error()).mockRejectedValueOnce(new Error()).mockResolvedValueOnce(undefined)
const dstPath = path.join(path.sep, 'missing', 'parent', 'destination.txt')
await expect(isCrossDevice('/source.txt', dstPath)).resolves.toBe(true)
diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.ts
index 988a9cf0..1c21bc30 100644
--- a/backend/src/applications/files/utils/files.ts
+++ b/backend/src/applications/files/utils/files.ts
@@ -67,7 +67,10 @@ export function checkFileName(fPath: string): string {
}
export function isPathExists(rPath: string): Promise<boolean> {
- return fse.pathExists(rPath)
+ return fs.access(rPath).then(
+ () => true,
+ () => false
+ )
}
async function existingParentPath(rPath: string): Promise<string> {
@@ -224,7 +227,7 @@ export function genUniqHashFromFileDBProps(dbFile: FileDBProps) {
export function removeFiles(rPath: string): Promise<void> {
// if the file does not exist, no error is thrown
- return fse.remove(rPath)
+ return fs.rm(rPath, { force: true, recursive: true })
}
export async function getProps(rPath: string, fPath?: string, isDir?: boolean): Promise<FileProps> {
@@ -247,6 +250,43 @@ export function touchFile(rPath: string, mtime?: number): Promise<void> {
return fs.utimes(rPath, mtime, mtime)
}
+const REFLINK_UNAVAILABLE_ERROR_CODES = new Set(['EAGAIN', 'EINVAL', 'ENOSYS', 'ENOTSUP', 'ENOTTY', 'EOPNOTSUPP', 'EXDEV'])
+
+export async function tryReflink(srcPath: string, dstPath: string): Promise<boolean> {
+ // A reflink is a copy-on-write clone that shares data blocks and allocates new ones only for modified regions.
+ try {
+ // FORCE lets the caller distinguish a real reflink from Node's transparent byte-copy fallback.
+ await fs.copyFile(srcPath, dstPath, fs.constants.COPYFILE_FICLONE_FORCE)
+ return true
+ } catch (error) {
+ const errorCode = (error as NodeJS.ErrnoException)?.code
+ if (errorCode && REFLINK_UNAVAILABLE_ERROR_CODES.has(errorCode)) {
+ return false
+ }
+ throw error
+ }
+}
+
+async function checkCopyParentPaths(srcPath: string, dstPath: string): Promise<void> {
+ const srcStats = await fs.lstat(srcPath, { bigint: true })
+ if (!srcStats.isDirectory()) return
+
+ const srcParentPath = path.resolve(path.dirname(srcPath))
+ let dstParentPath = path.resolve(path.dirname(dstPath))
+
+ while (dstParentPath !== srcParentPath && dstParentPath !== path.parse(dstParentPath).root) {
+ try {
+ const dstParentStats = await fs.stat(dstParentPath, { bigint: true })
+ if (srcStats.ino === dstParentStats.ino && srcStats.dev === dstParentStats.dev) {
+ throw new Error(`Cannot copy '${srcPath}' to a subdirectory of itself, '${dstPath}'.`)
+ }
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') throw error
+ }
+ dstParentPath = path.dirname(dstParentPath)
+ }
+}
+
export async function copyFiles(srcPath: string, dstPath: string, overwrite = false, recursive = true, preserveTimestamps = true): Promise<void> {
/*
If src is a directory it will copy everything inside of this directory, not the entire directory itself
@@ -260,11 +300,26 @@ export async function copyFiles(srcPath: string, dstPath: string, overwrite = fa
}
} else {
const resolvedSrcPath = path.resolve(srcPath)
- await fse.copy(srcPath, dstPath, {
- overwrite,
- preserveTimestamps,
- filter: (entryPath) => path.resolve(entryPath) === resolvedSrcPath || !isInternalTemporaryEntry(path.basename(entryPath))
- })
+ const filter = (entryPath: string): boolean => path.resolve(entryPath) === resolvedSrcPath || !isInternalTemporaryEntry(path.basename(entryPath))
+
+ await checkCopyParentPaths(srcPath, dstPath)
+ try {
+ await fs.cp(srcPath, dstPath, {
+ dereference: false,
+ force: overwrite,
+ mode: fs.constants.COPYFILE_FICLONE,
+ preserveTimestamps,
+ recursive,
+ // Match fs-extra: keep relative symlink targets relative instead of resolving them against the source directory.
+ verbatimSymlinks: true,
+ filter
+ })
+ } catch (error) {
+ // TODO: Remove this fallback once https://github.com/nodejs/node/issues/65097 is fixed in every supported Node release.
+ // Reproduced with fs.promises.cp on Node 24.20.0; fs-extra remains necessary for this case and for moveFiles' EXDEV fallback.
+ if ((error as NodeJS.ErrnoException)?.code !== 'ERR_FS_CP_EINVAL') throw error
+ await fse.copy(srcPath, dstPath, { overwrite, preserveTimestamps, filter })
+ }
}
}