Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

fix(folder-ingestion): allow cross docker volume file moving (#493)

Corentin Thomasseted4d7e4a00b2
diff --git a/.changeset/huge-forks-build.md b/.changeset/huge-forks-build.md
new file mode 100644
index 00000000..19c1bb93
--- /dev/null
+++ b/.changeset/huge-forks-build.md
@@ -0,0 +1,5 @@
+---
+"@papra/app-server": patch
+---
+
+Fix to allow cross docker volume file moving when consumption is done
diff --git a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
index 4480903d..d89ad8c5 100644
--- a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
+++ b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
@@ -3,7 +3,7 @@ import type { FsNative } from './fs.services';
import { memfs } from 'memfs';
import { createFsServices } from './fs.services';
-export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
+export function buildInMemoryFs(volume: NestedDirectoryJSON) {
const { vol } = memfs(volume);
const fs = {
@@ -12,7 +12,16 @@ export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
} as FsNative;
return {
+ fs,
getFsState: () => vol.toJSON(),
+ };
+}
+
+export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
+ const { fs, getFsState } = buildInMemoryFs(volume);
+
+ return {
+ getFsState,
fs: createFsServices({ fs }),
};
}
diff --git a/apps/papra-server/src/modules/shared/fs/fs.models.ts b/apps/papra-server/src/modules/shared/fs/fs.models.ts
new file mode 100644
index 00000000..ce9b9a9c
--- /dev/null
+++ b/apps/papra-server/src/modules/shared/fs/fs.models.ts
@@ -0,0 +1,12 @@
+import { isNil } from '../utils';
+
+export function isCrossDeviceError({ error }: { error: Error & { code?: unknown } }) {
+ if (isNil(error.code) || typeof error.code !== 'string') {
+ return false;
+ }
+
+ return [
+ 'EXDEV', // Linux based OS (see `man rename`)
+ 'ERROR_NOT_SAME_DEVICE', // Windows
+ ].includes(error.code);
+}
diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.test.ts b/apps/papra-server/src/modules/shared/fs/fs.services.test.ts
new file mode 100644
index 00000000..0d4ee9fe
--- /dev/null
+++ b/apps/papra-server/src/modules/shared/fs/fs.services.test.ts
@@ -0,0 +1,45 @@
+import { describe, expect, test } from 'vitest';
+import { buildInMemoryFs } from './fs.in-memory';
+import { moveFile } from './fs.services';
+
+describe('fs services', () => {
+ describe('moveFile', () => {
+ test('moves a file from the source path to the destination path', async () => {
+ const { fs, getFsState } = buildInMemoryFs({
+ '/file.txt': 'test content',
+ });
+
+ await moveFile({
+ sourceFilePath: '/file.txt',
+ destinationFilePath: '/renamed.txt',
+ fs,
+ });
+
+ expect(getFsState()).to.eql({
+ '/renamed.txt': 'test content',
+ });
+ });
+
+ test('if the destination file is in a different partition or disk, or a different docker volume, the underlying rename operation fails with an EXDEV error, so we fallback to copy + delete the source file', async () => {
+ const { fs, getFsState } = buildInMemoryFs({
+ '/file.txt': 'test content',
+ });
+
+ await moveFile({
+ sourceFilePath: '/file.txt',
+ destinationFilePath: '/renamed.txt',
+ fs: {
+ ...fs,
+ rename: async () => {
+ // Simulate an EXDEV error
+ throw Object.assign(new Error('EXDEV'), { code: 'EXDEV' });
+ },
+ },
+ });
+
+ expect(getFsState()).to.eql({
+ '/renamed.txt': 'test content',
+ });
+ });
+ });
+});
diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.ts b/apps/papra-server/src/modules/shared/fs/fs.services.ts
index 0290c5f5..55339fe1 100644
--- a/apps/papra-server/src/modules/shared/fs/fs.services.ts
+++ b/apps/papra-server/src/modules/shared/fs/fs.services.ts
@@ -2,8 +2,9 @@ import type { Readable } from 'node:stream';
import { Buffer } from 'node:buffer';
import fsSyncNative from 'node:fs';
import fsPromisesNative from 'node:fs/promises';
-import { injectArguments } from '@corentinth/chisels';
+import { injectArguments, safely } from '@corentinth/chisels';
import { pick } from 'lodash-es';
+import { isCrossDeviceError } from './fs.models';
// what we use from the native fs module
export type FsNative = {
@@ -13,12 +14,13 @@ export type FsNative = {
stat: (path: string) => Promise<{ size: number }>;
readFile: (path: string) => Promise<Buffer>;
access: (path: string, mode: number) => Promise<void>;
+ copyFile: (sourcePath: string, destinationPath: string) => Promise<void>;
constants: { F_OK: number };
createReadStream: (path: string) => Readable;
};
const fsNative = {
- ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat'),
+ ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat', 'copyFile'),
createReadStream: fsSyncNative.createReadStream.bind(fsSyncNative) as (filePath: string) => Readable,
} as FsNative;
@@ -66,7 +68,19 @@ export async function deleteFile({ filePath, fs = fsNative }: { filePath: string
}
export async function moveFile({ sourceFilePath, destinationFilePath, fs = fsNative }: { sourceFilePath: string; destinationFilePath: string; fs?: FsNative }) {
- await fs.rename(sourceFilePath, destinationFilePath);
+ const [, error] = await safely(fs.rename(sourceFilePath, destinationFilePath));
+
+ // With different docker volumes, the rename operation fails with an EXDEV error,
+ // so we fallback to copy and delete the source file
+ if (error && isCrossDeviceError({ error })) {
+ await fs.copyFile(sourceFilePath, destinationFilePath);
+ await fs.unlink(sourceFilePath);
+ return;
+ }
+
+ if (error) {
+ throw error;
+ }
}
export async function readFile({ filePath, fs = fsNative }: { filePath: string; fs?: FsNative }) {