Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

refactor(ingestion-folders): use file streaming instead of loading in ram (#475)

Corentin Thomassetea9d90d6cff6
diff --git a/.changeset/fuzzy-clocks-sell.md b/.changeset/fuzzy-clocks-sell.md
new file mode 100644
index 00000000..a26bca79
--- /dev/null
+++ b/.changeset/fuzzy-clocks-sell.md
@@ -0,0 +1,5 @@
+---
+"@papra/app-server": patch
+---
+
+Use node file streams in ingestion folder for smaller RAM footprint
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts
index 5ec19650..49acd726 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts
@@ -1,35 +1,30 @@
-import { Buffer } from 'node:buffer';
+import { Readable } from 'node:stream';
import { describe, expect, test } from 'vitest';
+import { createReadableStream } from '../shared/streams/readable-stream';
import { getFile } from './ingestion-folders.services';
describe('ingestion-folders services', () => {
describe('getFile', () => {
- const readFile = async () => Buffer.from('test');
-
test('reads a file from the fs and returns a File instance', async () => {
- const { file } = await getFile({
+ const { fileStream, mimeType, fileName } = getFile({
filePath: 'test.txt',
- fs: { readFile },
+ fs: { createReadStream: () => createReadableStream({ content: 'test' }) },
});
- expect(file).instanceOf(File);
- expect(file.type).to.equal('text/plain');
- expect(file.name).to.equal('test.txt');
- expect(file.size).to.equal(4);
- expect(await file.text()).to.equal('test');
+ expect(fileStream).instanceOf(Readable);
+ expect(mimeType).to.equal('text/plain');
+ expect(fileName).to.equal('test.txt');
});
test('a file with a weird extension is considered a octet-stream', async () => {
- const { file } = await getFile({
+ const { fileStream, mimeType, fileName } = getFile({
filePath: 'test.weird',
- fs: { readFile },
+ fs: { createReadStream: () => createReadableStream({ content: 'test' }) },
});
- expect(file).instanceOf(File);
- expect(file.type).to.equal('application/octet-stream');
- expect(file.name).to.equal('test.weird');
- expect(file.size).to.equal(4);
- expect(await file.text()).to.equal('test');
+ expect(fileStream).instanceOf(Readable);
+ expect(mimeType).to.equal('application/octet-stream');
+ expect(fileName).to.equal('test.weird');
});
});
});
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts
index 488be8ed..11073fb5 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts
@@ -7,20 +7,19 @@ import {
createFsServices,
} from '../shared/fs/fs.services';
-export async function getFile({
+export function getFile({
filePath,
fs = createFsServices(),
}: {
filePath: string;
- fs?: Pick<FsServices, 'readFile'>;
+ fs?: Pick<FsServices, 'createReadStream'>;
}) {
- const buffer = await fs.readFile({ filePath });
+ const fileStream = fs.createReadStream({ filePath });
// lookup returns false if the mime type is not found
const lookedUpMimeType = mime.lookup(filePath);
const mimeType = lookedUpMimeType === false ? 'application/octet-stream' : lookedUpMimeType;
const { base: fileName } = parse(filePath);
- const file = new File([buffer], fileName, { type: mimeType });
- return { file };
+ return { fileStream, mimeType, fileName };
}
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts
index ccdbe9c3..62d97b28 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts
@@ -1,6 +1,4 @@
-import type { FsNative } from '../shared/fs/fs.services';
import { safely } from '@corentinth/chisels';
-import { memfs } from 'memfs';
import { describe, expect, test } from 'vitest';
import { createInMemoryDatabase } from '../app/database/database.test-utils';
import { overrideConfig } from '../config/config.test-utils';
@@ -9,7 +7,6 @@ import { createDocumentCreationUsecase } from '../documents/documents.usecases';
import { inMemoryStorageDriverFactory } from '../documents/storage/drivers/memory/memory.storage-driver';
import { createOrganizationsRepository } from '../organizations/organizations.repository';
import { createInMemoryFsServices } from '../shared/fs/fs.in-memory';
-import { createFsServices } from '../shared/fs/fs.services';
import { createTestLogger } from '../shared/logger/logger.test-utils';
import { createInMemoryTaskServices } from '../tasks/tasks.test-utils';
import { createInvalidPostProcessingStrategyError } from './ingestion-folders.errors';
@@ -44,7 +41,7 @@ describe('ingestion-folders usecases', () => {
let documentIdIndex = 1;
const generateDocumentId = () => `doc_${documentIdIndex++}`;
- const { vol } = memfs({
+ const { fs, getFsState } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -54,7 +51,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
});
@@ -84,7 +81,7 @@ describe('ingestion-folders usecases', () => {
expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');
// Check FS, ensure the file has been moved to the done folder
- expect(vol.toJSON()).to.deep.equal({
+ expect(getFsState()).to.deep.equal({
'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum',
});
@@ -146,7 +143,7 @@ describe('ingestion-folders usecases', () => {
let documentIdIndex = 1;
const generateDocumentId = () => `doc_${documentIdIndex++}`;
- const { vol } = memfs({
+ const { fs, getFsState } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -156,7 +153,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
});
@@ -186,7 +183,7 @@ describe('ingestion-folders usecases', () => {
expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');
// Check FS, ensure the file has been moved to the done folder
- expect(vol.toJSON()).to.deep.equal({
+ expect(getFsState()).to.deep.equal({
'/apps/papra/ingestion/org_111111111111111111111111': null,
});
@@ -249,7 +246,7 @@ describe('ingestion-folders usecases', () => {
let documentIdIndex = 1;
const generateDocumentId = () => `doc_${documentIdIndex++}`;
- const { vol } = memfs({
+ const { fs, getFsState } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -259,7 +256,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
}));
@@ -291,7 +288,7 @@ describe('ingestion-folders usecases', () => {
expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');
// Check FS, ensure the file is still in the ingestion folder
- expect(vol.toJSON()).to.deep.equal({
+ expect(getFsState()).to.deep.equal({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
});
@@ -322,7 +319,7 @@ describe('ingestion-folders usecases', () => {
let documentIdIndex = 1;
const generateDocumentId = () => `doc_${documentIdIndex++}`;
- const { vol } = memfs({
+ const { fs } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -333,8 +330,8 @@ describe('ingestion-folders usecases', () => {
organizationsRepository,
logger,
fs: {
- ...createFsServices({ fs: vol.promises as unknown as FsNative }),
- readFile: async () => {
+ ...fs,
+ createReadStream: () => {
throw new Error('File not found');
},
},
@@ -384,7 +381,7 @@ describe('ingestion-folders usecases', () => {
},
});
- const { vol } = memfs({
+ const { fs } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum',
'/apps/papra/ingestion/org_111111111111111111111111/error/world.md': 'dolor sit amet',
});
@@ -395,7 +392,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: async () => expect.fail('Document should not be created'),
});
@@ -405,7 +402,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: async () => expect.fail('Document should not be created'),
});
@@ -467,7 +464,7 @@ describe('ingestion-folders usecases', () => {
},
});
- const { vol } = memfs({
+ const { fs, getFsState } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -477,7 +474,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),
});
@@ -488,7 +485,7 @@ describe('ingestion-folders usecases', () => {
expect(documents[0]?.id).to.equal('doc_1');
// Check fs
- expect(vol.toJSON()).to.deep.equal({
+ expect(getFsState()).to.deep.equal({
'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum',
});
@@ -541,7 +538,7 @@ describe('ingestion-folders usecases', () => {
},
});
- const { vol } = memfs({
+ const { fs, getFsState } = createInMemoryFsServices({
'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',
});
@@ -551,7 +548,7 @@ describe('ingestion-folders usecases', () => {
config,
organizationsRepository,
logger,
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs,
createDocument: async () => {
throw new Error('Document creation failed');
},
@@ -564,7 +561,7 @@ describe('ingestion-folders usecases', () => {
expect(documents[0]?.id).to.equal('doc_1');
// Check fs
- expect(vol.toJSON()).to.deep.equal({
+ expect(getFsState()).to.deep.equal({
'/apps/papra/ingestion/org_111111111111111111111111/error/hello.md': 'lorem ipsum',
});
diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
index 12619779..e84edefc 100644
--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts
@@ -8,7 +8,7 @@ import type { FsServices } from '../shared/fs/fs.services';
import type { Logger } from '../shared/logger/logger';
import type { TaskServices } from '../tasks/tasks.services';
import { isAbsolute, join, parse } from 'node:path';
-import { safely } from '@corentinth/chisels';
+import { safely, safelySync } from '@corentinth/chisels';
import chokidar from 'chokidar';
import { uniq } from 'lodash-es';
import PQueue from 'p-queue';
@@ -20,7 +20,6 @@ import { isErrorWithCode } from '../shared/errors/errors';
import { createFsServices } from '../shared/fs/fs.services';
import { createLogger } from '../shared/logger/logger';
import { getRootDirPath } from '../shared/path';
-import { fileToReadableStream } from '../shared/streams/readable-stream';
import { isNil } from '../shared/utils';
import { addTimestampToFilename, getAbsolutePathFromFolderRelativeToOrganizationIngestionFolder, getOrganizationIdFromFilePath, isFileInDoneFolder, isFileInErrorFolder, normalizeFilePathToIngestionFolder } from './ingestion-folder.models';
import { createInvalidPostProcessingStrategyError } from './ingestion-folders.errors';
@@ -113,14 +112,14 @@ export async function processFile({
const { postProcessing: { moveToFolderPath: doneFolder }, errorFolder } = config.ingestionFolder;
// Get the file from the ingestion folder as a File Instance
- const [getFileResult, getFileError] = await safely(getFile({ filePath, fs }));
+ const [getFileResult, getFileError] = safelySync(() => getFile({ filePath, fs }));
if (getFileError) {
logger.error({ filePath, error: getFileError }, 'Error reading file');
return;
}
- const { file } = getFileResult;
+ const { fileStream, mimeType, fileName } = getFileResult;
const { organizationId } = await getFileOrganizationId({ filePath, ingestionFolderPath, organizationsRepository });
@@ -143,9 +142,9 @@ export async function processFile({
// TODO: switch to native stream
const [result, error] = await safely(createDocument({
- fileStream: fileToReadableStream(file),
- fileName: file.name,
- mimeType: file.type,
+ fileStream,
+ fileName,
+ mimeType,
organizationId,
}));
diff --git a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
index b0f2bb1e..4480903d 100644
--- a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
+++ b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts
@@ -6,8 +6,13 @@ import { createFsServices } from './fs.services';
export function createInMemoryFsServices(volume: NestedDirectoryJSON) {
const { vol } = memfs(volume);
+ const fs = {
+ ...vol.promises,
+ createReadStream: (filePath: string) => vol.createReadStream(filePath),
+ } as FsNative;
+
return {
getFsState: () => vol.toJSON(),
- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),
+ fs: createFsServices({ fs }),
};
}
diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.ts b/apps/papra-server/src/modules/shared/fs/fs.services.ts
index a55930b5..0290c5f5 100644
--- a/apps/papra-server/src/modules/shared/fs/fs.services.ts
+++ b/apps/papra-server/src/modules/shared/fs/fs.services.ts
@@ -1,8 +1,26 @@
+import type { Readable } from 'node:stream';
import { Buffer } from 'node:buffer';
-import fsNative from 'node:fs/promises';
+import fsSyncNative from 'node:fs';
+import fsPromisesNative from 'node:fs/promises';
import { injectArguments } from '@corentinth/chisels';
+import { pick } from 'lodash-es';
-export type FsNative = Pick<typeof fsNative, 'mkdir' | 'unlink' | 'rename' | 'readFile' | 'stat' | 'access' | 'constants'>;
+// what we use from the native fs module
+export type FsNative = {
+ mkdir: (path: string, options: { recursive: true }) => Promise<void>;
+ unlink: (path: string) => Promise<void>;
+ rename: (oldPath: string, newPath: string) => Promise<void>;
+ stat: (path: string) => Promise<{ size: number }>;
+ readFile: (path: string) => Promise<Buffer>;
+ access: (path: string, mode: number) => Promise<void>;
+ constants: { F_OK: number };
+ createReadStream: (path: string) => Readable;
+};
+
+const fsNative = {
+ ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat'),
+ createReadStream: fsSyncNative.createReadStream.bind(fsSyncNative) as (filePath: string) => Readable,
+} as FsNative;
export type FsServices = ReturnType<typeof createFsServices>;
export function createFsServices({ fs = fsNative }: { fs?: FsNative } = {}) {
@@ -13,6 +31,7 @@ export function createFsServices({ fs = fsNative }: { fs?: FsNative } = {}) {
deleteFile,
moveFile,
readFile,
+ createReadStream,
areFilesContentIdentical,
},
{
@@ -54,6 +73,10 @@ export async function readFile({ filePath, fs = fsNative }: { filePath: string;
return fs.readFile(filePath);
}
+export function createReadStream({ filePath, fs = fsNative }: { filePath: string; fs?: FsNative }) {
+ return fs.createReadStream(filePath);
+}
+
export async function areFilesContentIdentical({ file1, file2, fs = fsNative }: { file1: string; file2: string; fs?: FsNative }): Promise<boolean> {
try {
// Check if file sizes are different (quick check before comparing content)