The minimalistic document archiving platform.
refactor(ingestion-folders): use file streaming instead of loading in ram (#475)
Corentin Thomassetea9d90d6cff6
diff --git a/.changeset/fuzzy-clocks-sell.md b/.changeset/fuzzy-clocks-sell.mdnew file mode 100644index 00000000..a26bca79--- /dev/null+++ b/.changeset/fuzzy-clocks-sell.md@@ -0,0 +1,5 @@+---+"@papra/app-server": patch+---++Use node file streams in ingestion folder for smaller RAM footprintdiff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.tsindex 5ec19650..49acd726 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.test.ts@@ -1,35 +1,30 @@-import { Buffer } from 'node:buffer';+import { Readable } from 'node:stream';import { describe, expect, test } from 'vitest';+import { createReadableStream } from '../shared/streams/readable-stream';import { getFile } from './ingestion-folders.services';describe('ingestion-folders services', () => {describe('getFile', () => {- const readFile = async () => Buffer.from('test');-test('reads a file from the fs and returns a File instance', async () => {- const { file } = await getFile({+ const { fileStream, mimeType, fileName } = getFile({filePath: 'test.txt',- fs: { readFile },+ fs: { createReadStream: () => createReadableStream({ content: 'test' }) },});- expect(file).instanceOf(File);- expect(file.type).to.equal('text/plain');- expect(file.name).to.equal('test.txt');- expect(file.size).to.equal(4);- expect(await file.text()).to.equal('test');+ expect(fileStream).instanceOf(Readable);+ expect(mimeType).to.equal('text/plain');+ expect(fileName).to.equal('test.txt');});test('a file with a weird extension is considered a octet-stream', async () => {- const { file } = await getFile({+ const { fileStream, mimeType, fileName } = getFile({filePath: 'test.weird',- fs: { readFile },+ fs: { createReadStream: () => createReadableStream({ content: 'test' }) },});- expect(file).instanceOf(File);- expect(file.type).to.equal('application/octet-stream');- expect(file.name).to.equal('test.weird');- expect(file.size).to.equal(4);- expect(await file.text()).to.equal('test');+ expect(fileStream).instanceOf(Readable);+ expect(mimeType).to.equal('application/octet-stream');+ expect(fileName).to.equal('test.weird');});});});diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.tsindex 488be8ed..11073fb5 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.services.ts@@ -7,20 +7,19 @@ import {createFsServices,} from '../shared/fs/fs.services';-export async function getFile({+export function getFile({filePath,fs = createFsServices(),}: {filePath: string;- fs?: Pick<FsServices, 'readFile'>;+ fs?: Pick<FsServices, 'createReadStream'>;}) {- const buffer = await fs.readFile({ filePath });+ const fileStream = fs.createReadStream({ filePath });// lookup returns false if the mime type is not foundconst lookedUpMimeType = mime.lookup(filePath);const mimeType = lookedUpMimeType === false ? 'application/octet-stream' : lookedUpMimeType;const { base: fileName } = parse(filePath);- const file = new File([buffer], fileName, { type: mimeType });- return { file };+ return { fileStream, mimeType, fileName };}diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.tsindex ccdbe9c3..62d97b28 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts@@ -1,6 +1,4 @@-import type { FsNative } from '../shared/fs/fs.services';import { safely } from '@corentinth/chisels';-import { memfs } from 'memfs';import { describe, expect, test } from 'vitest';import { createInMemoryDatabase } from '../app/database/database.test-utils';import { overrideConfig } from '../config/config.test-utils';@@ -9,7 +7,6 @@ import { createDocumentCreationUsecase } from '../documents/documents.usecases';import { inMemoryStorageDriverFactory } from '../documents/storage/drivers/memory/memory.storage-driver';import { createOrganizationsRepository } from '../organizations/organizations.repository';import { createInMemoryFsServices } from '../shared/fs/fs.in-memory';-import { createFsServices } from '../shared/fs/fs.services';import { createTestLogger } from '../shared/logger/logger.test-utils';import { createInMemoryTaskServices } from '../tasks/tasks.test-utils';import { createInvalidPostProcessingStrategyError } from './ingestion-folders.errors';@@ -44,7 +41,7 @@ describe('ingestion-folders usecases', () => {let documentIdIndex = 1;const generateDocumentId = () => `doc_${documentIdIndex++}`;- const { vol } = memfs({+ const { fs, getFsState } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -54,7 +51,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});@@ -84,7 +81,7 @@ describe('ingestion-folders usecases', () => {expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');// Check FS, ensure the file has been moved to the done folder- expect(vol.toJSON()).to.deep.equal({+ expect(getFsState()).to.deep.equal({'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum',});@@ -146,7 +143,7 @@ describe('ingestion-folders usecases', () => {let documentIdIndex = 1;const generateDocumentId = () => `doc_${documentIdIndex++}`;- const { vol } = memfs({+ const { fs, getFsState } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -156,7 +153,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});@@ -186,7 +183,7 @@ describe('ingestion-folders usecases', () => {expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');// Check FS, ensure the file has been moved to the done folder- expect(vol.toJSON()).to.deep.equal({+ expect(getFsState()).to.deep.equal({'/apps/papra/ingestion/org_111111111111111111111111': null,});@@ -249,7 +246,7 @@ describe('ingestion-folders usecases', () => {let documentIdIndex = 1;const generateDocumentId = () => `doc_${documentIdIndex++}`;- const { vol } = memfs({+ const { fs, getFsState } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -259,7 +256,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),}));@@ -291,7 +288,7 @@ describe('ingestion-folders usecases', () => {expect(file!.content.toString('utf-8')).to.equal('lorem ipsum');// Check FS, ensure the file is still in the ingestion folder- expect(vol.toJSON()).to.deep.equal({+ expect(getFsState()).to.deep.equal({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});});@@ -322,7 +319,7 @@ describe('ingestion-folders usecases', () => {let documentIdIndex = 1;const generateDocumentId = () => `doc_${documentIdIndex++}`;- const { vol } = memfs({+ const { fs } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -333,8 +330,8 @@ describe('ingestion-folders usecases', () => {organizationsRepository,logger,fs: {- ...createFsServices({ fs: vol.promises as unknown as FsNative }),- readFile: async () => {+ ...fs,+ createReadStream: () => {throw new Error('File not found');},},@@ -384,7 +381,7 @@ describe('ingestion-folders usecases', () => {},});- const { vol } = memfs({+ const { fs } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum','/apps/papra/ingestion/org_111111111111111111111111/error/world.md': 'dolor sit amet',});@@ -395,7 +392,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: async () => expect.fail('Document should not be created'),});@@ -405,7 +402,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: async () => expect.fail('Document should not be created'),});@@ -467,7 +464,7 @@ describe('ingestion-folders usecases', () => {},});- const { vol } = memfs({+ const { fs, getFsState } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -477,7 +474,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});@@ -488,7 +485,7 @@ describe('ingestion-folders usecases', () => {expect(documents[0]?.id).to.equal('doc_1');// Check fs- expect(vol.toJSON()).to.deep.equal({+ expect(getFsState()).to.deep.equal({'/apps/papra/ingestion/org_111111111111111111111111/done/hello.md': 'lorem ipsum',});@@ -541,7 +538,7 @@ describe('ingestion-folders usecases', () => {},});- const { vol } = memfs({+ const { fs, getFsState } = createInMemoryFsServices({'/apps/papra/ingestion/org_111111111111111111111111/hello.md': 'lorem ipsum',});@@ -551,7 +548,7 @@ describe('ingestion-folders usecases', () => {config,organizationsRepository,logger,- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs,createDocument: async () => {throw new Error('Document creation failed');},@@ -564,7 +561,7 @@ describe('ingestion-folders usecases', () => {expect(documents[0]?.id).to.equal('doc_1');// Check fs- expect(vol.toJSON()).to.deep.equal({+ expect(getFsState()).to.deep.equal({'/apps/papra/ingestion/org_111111111111111111111111/error/hello.md': 'lorem ipsum',});diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.tsindex 12619779..e84edefc 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts@@ -8,7 +8,7 @@ import type { FsServices } from '../shared/fs/fs.services';import type { Logger } from '../shared/logger/logger';import type { TaskServices } from '../tasks/tasks.services';import { isAbsolute, join, parse } from 'node:path';-import { safely } from '@corentinth/chisels';+import { safely, safelySync } from '@corentinth/chisels';import chokidar from 'chokidar';import { uniq } from 'lodash-es';import PQueue from 'p-queue';@@ -20,7 +20,6 @@ import { isErrorWithCode } from '../shared/errors/errors';import { createFsServices } from '../shared/fs/fs.services';import { createLogger } from '../shared/logger/logger';import { getRootDirPath } from '../shared/path';-import { fileToReadableStream } from '../shared/streams/readable-stream';import { isNil } from '../shared/utils';import { addTimestampToFilename, getAbsolutePathFromFolderRelativeToOrganizationIngestionFolder, getOrganizationIdFromFilePath, isFileInDoneFolder, isFileInErrorFolder, normalizeFilePathToIngestionFolder } from './ingestion-folder.models';import { createInvalidPostProcessingStrategyError } from './ingestion-folders.errors';@@ -113,14 +112,14 @@ export async function processFile({const { postProcessing: { moveToFolderPath: doneFolder }, errorFolder } = config.ingestionFolder;// Get the file from the ingestion folder as a File Instance- const [getFileResult, getFileError] = await safely(getFile({ filePath, fs }));+ const [getFileResult, getFileError] = safelySync(() => getFile({ filePath, fs }));if (getFileError) {logger.error({ filePath, error: getFileError }, 'Error reading file');return;}- const { file } = getFileResult;+ const { fileStream, mimeType, fileName } = getFileResult;const { organizationId } = await getFileOrganizationId({ filePath, ingestionFolderPath, organizationsRepository });@@ -143,9 +142,9 @@ export async function processFile({// TODO: switch to native streamconst [result, error] = await safely(createDocument({- fileStream: fileToReadableStream(file),- fileName: file.name,- mimeType: file.type,+ fileStream,+ fileName,+ mimeType,organizationId,}));diff --git a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts b/apps/papra-server/src/modules/shared/fs/fs.in-memory.tsindex b0f2bb1e..4480903d 100644--- a/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts+++ b/apps/papra-server/src/modules/shared/fs/fs.in-memory.ts@@ -6,8 +6,13 @@ import { createFsServices } from './fs.services';export function createInMemoryFsServices(volume: NestedDirectoryJSON) {const { vol } = memfs(volume);+ const fs = {+ ...vol.promises,+ createReadStream: (filePath: string) => vol.createReadStream(filePath),+ } as FsNative;+return {getFsState: () => vol.toJSON(),- fs: createFsServices({ fs: vol.promises as unknown as FsNative }),+ fs: createFsServices({ fs }),};}diff --git a/apps/papra-server/src/modules/shared/fs/fs.services.ts b/apps/papra-server/src/modules/shared/fs/fs.services.tsindex a55930b5..0290c5f5 100644--- a/apps/papra-server/src/modules/shared/fs/fs.services.ts+++ b/apps/papra-server/src/modules/shared/fs/fs.services.ts@@ -1,8 +1,26 @@+import type { Readable } from 'node:stream';import { Buffer } from 'node:buffer';-import fsNative from 'node:fs/promises';+import fsSyncNative from 'node:fs';+import fsPromisesNative from 'node:fs/promises';import { injectArguments } from '@corentinth/chisels';+import { pick } from 'lodash-es';-export type FsNative = Pick<typeof fsNative, 'mkdir' | 'unlink' | 'rename' | 'readFile' | 'stat' | 'access' | 'constants'>;+// what we use from the native fs module+export type FsNative = {+ mkdir: (path: string, options: { recursive: true }) => Promise<void>;+ unlink: (path: string) => Promise<void>;+ rename: (oldPath: string, newPath: string) => Promise<void>;+ stat: (path: string) => Promise<{ size: number }>;+ readFile: (path: string) => Promise<Buffer>;+ access: (path: string, mode: number) => Promise<void>;+ constants: { F_OK: number };+ createReadStream: (path: string) => Readable;+};++const fsNative = {+ ...pick(fsPromisesNative, 'mkdir', 'unlink', 'rename', 'readFile', 'access', 'constants', 'stat'),+ createReadStream: fsSyncNative.createReadStream.bind(fsSyncNative) as (filePath: string) => Readable,+} as FsNative;export type FsServices = ReturnType<typeof createFsServices>;export function createFsServices({ fs = fsNative }: { fs?: FsNative } = {}) {@@ -13,6 +31,7 @@ export function createFsServices({ fs = fsNative }: { fs?: FsNative } = {}) {deleteFile,moveFile,readFile,+ createReadStream,areFilesContentIdentical,},{@@ -54,6 +73,10 @@ export async function readFile({ filePath, fs = fsNative }: { filePath: string;return fs.readFile(filePath);}+export function createReadStream({ filePath, fs = fsNative }: { filePath: string; fs?: FsNative }) {+ return fs.createReadStream(filePath);+}+export async function areFilesContentIdentical({ file1, file2, fs = fsNative }: { file1: string; file2: string; fs?: FsNative }): Promise<boolean> {try {// Check if file sizes are different (quick check before comparing content)