Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
perf(backend:files): use reflinks for copy operations
johavend0b4c2c8008b
diff --git a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.tsindex 20bbdcd1..85aa0166 100644--- a/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts+++ b/backend/src/applications/files/services/tasks/files-tasks-transfer.service.ts@@ -17,7 +17,8 @@ import {isInternalTemporaryEntry,isPathExists,removeFiles,- temporaryFilePath+ temporaryFilePath,+ tryReflink} from '../../utils/files'import { countDirEntriesAndSize, isCrossDeviceError } from '../../utils/tasks'import { SourceCleanupError } from '../../models/file-error'@@ -283,12 +284,18 @@ export class FilesTasksTransfer {} else if (stats.isSymbolicLink()) {await fs.symlink(await fs.readlink(srcPath), dstPath)} else {- const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })- const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })- if (onProgress) {- await pipeline(src, createProgressTransform(onProgress), dst, { signal })+ const reflinked = await tryReflink(srcPath, dstPath)+ if (reflinked) {+ signal.throwIfAborted()+ onProgress?.(stats.size)} else {- await pipeline(src, dst, { signal })+ const src = createReadStream(srcPath, { highWaterMark: DEFAULT_HIGH_WATER_MARK })+ const dst = createWriteStream(dstPath, { mode: stats.mode, highWaterMark: DEFAULT_HIGH_WATER_MARK })+ if (onProgress) {+ await pipeline(src, createProgressTransform(onProgress), dst, { signal })+ } else {+ await pipeline(src, dst, { signal })+ }}}if (!stats.isSymbolicLink()) {diff --git a/backend/src/applications/files/utils/files.spec.ts b/backend/src/applications/files/utils/files.spec.tsindex 8fed7238..8b39322e 100644--- a/backend/src/applications/files/utils/files.spec.ts+++ b/backend/src/applications/files/utils/files.spec.ts@@ -3,8 +3,6 @@ import fs, { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'import os from 'node:os'import path from 'node:path'import { Readable } from 'node:stream'-import fse from 'fs-extra'-import type { MockInstance } from 'vitest'import { FileError } from '../models/file-error'import { storageQuotaExceededError } from './errors'import {@@ -92,8 +90,7 @@ describe(isCrossDevice.name, () => {it('compares the source device with the nearest existing destination parent', async () => {vi.spyOn(fs, 'lstat').mockResolvedValueOnce({ dev: 1 } as any)const statSpy = vi.spyOn(fs, 'stat').mockResolvedValueOnce({ dev: 2 } as any)- const pathExistsSpy = vi.spyOn(fse, 'pathExists') as unknown as MockInstance<(path: string) => Promise<boolean>>- pathExistsSpy.mockResolvedValueOnce(false).mockResolvedValueOnce(false).mockResolvedValueOnce(true)+ vi.spyOn(fs, 'access').mockRejectedValueOnce(new Error()).mockRejectedValueOnce(new Error()).mockResolvedValueOnce(undefined)const dstPath = path.join(path.sep, 'missing', 'parent', 'destination.txt')await expect(isCrossDevice('/source.txt', dstPath)).resolves.toBe(true)diff --git a/backend/src/applications/files/utils/files.ts b/backend/src/applications/files/utils/files.tsindex 988a9cf0..1c21bc30 100644--- a/backend/src/applications/files/utils/files.ts+++ b/backend/src/applications/files/utils/files.ts@@ -67,7 +67,10 @@ export function checkFileName(fPath: string): string {}export function isPathExists(rPath: string): Promise<boolean> {- return fse.pathExists(rPath)+ return fs.access(rPath).then(+ () => true,+ () => false+ )}async function existingParentPath(rPath: string): Promise<string> {@@ -224,7 +227,7 @@ export function genUniqHashFromFileDBProps(dbFile: FileDBProps) {export function removeFiles(rPath: string): Promise<void> {// if the file does not exist, no error is thrown- return fse.remove(rPath)+ return fs.rm(rPath, { force: true, recursive: true })}export async function getProps(rPath: string, fPath?: string, isDir?: boolean): Promise<FileProps> {@@ -247,6 +250,43 @@ export function touchFile(rPath: string, mtime?: number): Promise<void> {return fs.utimes(rPath, mtime, mtime)}+const REFLINK_UNAVAILABLE_ERROR_CODES = new Set(['EAGAIN', 'EINVAL', 'ENOSYS', 'ENOTSUP', 'ENOTTY', 'EOPNOTSUPP', 'EXDEV'])++export async function tryReflink(srcPath: string, dstPath: string): Promise<boolean> {+ // A reflink is a copy-on-write clone that shares data blocks and allocates new ones only for modified regions.+ try {+ // FORCE lets the caller distinguish a real reflink from Node's transparent byte-copy fallback.+ await fs.copyFile(srcPath, dstPath, fs.constants.COPYFILE_FICLONE_FORCE)+ return true+ } catch (error) {+ const errorCode = (error as NodeJS.ErrnoException)?.code+ if (errorCode && REFLINK_UNAVAILABLE_ERROR_CODES.has(errorCode)) {+ return false+ }+ throw error+ }+}++async function checkCopyParentPaths(srcPath: string, dstPath: string): Promise<void> {+ const srcStats = await fs.lstat(srcPath, { bigint: true })+ if (!srcStats.isDirectory()) return++ const srcParentPath = path.resolve(path.dirname(srcPath))+ let dstParentPath = path.resolve(path.dirname(dstPath))++ while (dstParentPath !== srcParentPath && dstParentPath !== path.parse(dstParentPath).root) {+ try {+ const dstParentStats = await fs.stat(dstParentPath, { bigint: true })+ if (srcStats.ino === dstParentStats.ino && srcStats.dev === dstParentStats.dev) {+ throw new Error(`Cannot copy '${srcPath}' to a subdirectory of itself, '${dstPath}'.`)+ }+ } catch (error) {+ if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') throw error+ }+ dstParentPath = path.dirname(dstParentPath)+ }+}+export async function copyFiles(srcPath: string, dstPath: string, overwrite = false, recursive = true, preserveTimestamps = true): Promise<void> {/*If src is a directory it will copy everything inside of this directory, not the entire directory itself@@ -260,11 +300,26 @@ export async function copyFiles(srcPath: string, dstPath: string, overwrite = fa}} else {const resolvedSrcPath = path.resolve(srcPath)- await fse.copy(srcPath, dstPath, {- overwrite,- preserveTimestamps,- filter: (entryPath) => path.resolve(entryPath) === resolvedSrcPath || !isInternalTemporaryEntry(path.basename(entryPath))- })+ const filter = (entryPath: string): boolean => path.resolve(entryPath) === resolvedSrcPath || !isInternalTemporaryEntry(path.basename(entryPath))++ await checkCopyParentPaths(srcPath, dstPath)+ try {+ await fs.cp(srcPath, dstPath, {+ dereference: false,+ force: overwrite,+ mode: fs.constants.COPYFILE_FICLONE,+ preserveTimestamps,+ recursive,+ // Match fs-extra: keep relative symlink targets relative instead of resolving them against the source directory.+ verbatimSymlinks: true,+ filter+ })+ } catch (error) {+ // TODO: Remove this fallback once https://github.com/nodejs/node/issues/65097 is fixed in every supported Node release.+ // Reproduced with fs.promises.cp on Node 24.20.0; fs-extra remains necessary for this case and for moveFiles' EXDEV fallback.+ if ((error as NodeJS.ErrnoException)?.code !== 'ERR_FS_CP_EINVAL') throw error+ await fse.copy(srcPath, dstPath, { overwrite, preserveTimestamps, filter })+ }}}