The minimalistic document archiving platform.
fix(folder-ingestion): allow cross docker volume file moving (#493)
Corentin Thomasseted4d7e4a00b2
diff --git a/.changeset/huge-forks-build.md b/.changeset/huge-forks-build.mdnew file mode 100644index 00000000..19c1bb93--- /dev/null+++ b/.changeset/huge-forks-build.md@@ -0,0 +1,5 @@+---+"@papra/app-server": patch+---++Fix to allow cross docker volume file moving when consumption is donediff --git a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts b/apps/papra-server/src/modules/shared/fs/fs.in-memory.tsindex 4480903d..d89ad8c5 100644--- a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts+++ b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts@@ -3,7 +3,7 @@ import type { FsNative } from './fs.services';import { memfs } from 'memfs';import { createFsServices } from './fs.services';-export function createInMemoryFsServices(volume: NestedDirectoryJSON) {+export function buildInMemoryFs(volume: NestedDirectoryJSON) {const { vol } = memfs(volume);const fs = {@@ -12,7 +12,16 @@ export function createInMemoryFsServices(volume: NestedDirectoryJSON) {} as FsNative;return {+ fs,getFsState: () => vol.toJSON(),+ };+}++export function createInMemoryFsServices(volume: NestedDirectoryJSON) {+ const { fs, getFsState } = buildInMemoryFs(volume);++ return {+ getFsState,fs: createFsServices({ fs }),};}diff --git a/apps/papra-server/src/modules/shared/fs/fs.models.ts b/apps/papra-server/src/modules/shared/fs/fs.models.tsnew file mode 100644index 00000000..ce9b9a9c--- /dev/null+++ b/apps/papra-server/src/modules/shared/fs/fs.models.ts@@ -0,0 +1,12 @@+import { isNil } from '../utils';++export function isCrossDeviceError({ error }: { error: Error & { code?: unknown } }) {+ if (isNil(error.code) || typeof error.code !== 'string') {+ return false;+ }++ return [+ 'EXDEV', // Linux based OS (see `man rename`)+ 'ERROR_NOT_SAME_DEVICE', // Windows+ ].includes(error.code);+}diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.test.ts b/apps/papra-server/src/modules/shared/fs/fs.services.test.tsnew file mode 100644index 00000000..0d4ee9fe--- /dev/null+++ b/apps/papra-server/src/modules/shared/fs/fs.services.test.ts@@ -0,0 +1,45 @@+import { describe, expect, test } from 'vitest';+import { buildInMemoryFs } from './fs.in-memory';+import { moveFile } from './fs.services';++describe('fs services', () => {+ describe('moveFile', () => {+ test('moves a file from the source path to the destination path', async () => {+ const { fs, getFsState } = buildInMemoryFs({+ '/file.txt': 'test content',+ });++ await moveFile({+ sourceFilePath: '/file.txt',+ destinationFilePath: '/renamed.txt',+ fs,+ });++ expect(getFsState()).to.eql({+ '/renamed.txt': 'test content',+ });+ });++ test('if the destination file is in a different partition or disk, or a different docker volume, the underlying rename operation fails with an EXDEV error, so we fallback to copy + delete the source file', async () => {+ const { fs, getFsState } = buildInMemoryFs({+ '/file.txt': 'test content',+ });++ await moveFile({+ sourceFilePath: '/file.txt',+ destinationFilePath: '/renamed.txt',+ fs: {+ ...fs,+ rename: async () => {+ // Simulate an EXDEV error+ throw Object.assign(new Error('EXDEV'), { code: 'EXDEV' });+ },+ },+ });++ expect(getFsState()).to.eql({+ '/renamed.txt': 'test content',+ });+ });+ });+});diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.ts b/apps/papra-server/src/modules/shared/fs/fs.services.tsindex 0290c5f5..55339fe1 100644--- a/apps/papra-server/src/modules/shared/fs/fs.services.ts+++ b/apps/papra-server/src/modules/shared/fs/fs.services.ts@@ -2,8 +2,9 @@ import type { Readable } from 'node:stream';import { Buffer } from 'node:buffer';import fsSyncNative from 'node:fs';import fsPromisesNative from 'node:fs/promises';-import { injectArguments } from '@corentinth/chisels';+import { injectArguments, safely } from '@corentinth/chisels';import { pick } from 'lodash-es';+import { isCrossDeviceError } from './fs.models';// what we use from the native fs moduleexport type FsNative = {@@ -13,12 +14,13 @@ export type FsNative = {stat: (path: string) => Promise<{ size: number }>;readFile: (path: string) => Promise<Buffer>;access: (path: string, mode: number) => Promise<void>;+ copyFile: (sourcePath: string, destinationPath: string) => Promise<void>;constants: { F_OK: number };createReadStream: (path: string) => Readable;};const fsNative = {- ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat'),+ ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat', 'copyFile'),createReadStream: fsSyncNative.createReadStream.bind(fsSyncNative) as (filePath: string) => Readable,} as FsNative;@@ -66,7 +68,19 @@ export async function deleteFile({ filePath, fs = fsNative }: { filePath: string}export async function moveFile({ sourceFilePath, destinationFilePath, fs = fsNative }: { sourceFilePath: string; destinationFilePath: string; fs?: FsNative }) {- await fs.rename(sourceFilePath, destinationFilePath);+ const [, error] = await safely(fs.rename(sourceFilePath, destinationFilePath));++ // With different docker volumes, the rename operation fails with an EXDEV error,+ // so we fallback to copy and delete the source file+ if (error && isCrossDeviceError({ error })) {+ await fs.copyFile(sourceFilePath, destinationFilePath);+ await fs.unlink(sourceFilePath);+ return;+ }++ if (error) {+ throw error;+ }}export async function readFile({ filePath, fs = fsNative }: { filePath: string; fs?: FsNative }) {