The minimalistic document archiving platform.
refactor(documents): document content extraction is now async (#447)
Corentin Thomassetb5ccc135ba7f
diff --git a/.changeset/cyan-pots-begin.md b/.changeset/cyan-pots-begin.mdnew file mode 100644index 00000000..46c48e91--- /dev/null+++ b/.changeset/cyan-pots-begin.md@@ -0,0 +1,5 @@+---+"@papra/app-server": minor+---++The file content extraction (like OCR) is now done asynchronously by the task runnerdiff --git a/apps/papra-server/package.json b/apps/papra-server/package.jsonindex 1eda5a8d..9b0a72fa 100644--- a/apps/papra-server/package.json+++ b/apps/papra-server/package.json@@ -33,8 +33,8 @@"@aws-sdk/client-s3": "^3.835.0","@aws-sdk/lib-storage": "^3.835.0","@azure/storage-blob": "^12.27.0",- "@cadence-mq/core": "^0.1.0",- "@cadence-mq/driver-memory": "^0.1.0",+ "@cadence-mq/core": "^0.2.1",+ "@cadence-mq/driver-memory": "^0.2.0","@corentinth/chisels": "^1.3.1","@corentinth/friendly-ids": "^0.0.1","@crowlog/async-context-plugin": "^1.2.1",diff --git a/apps/papra-server/src/index.ts b/apps/papra-server/src/index.tsindex 2d492eee..0bdcc907 100644--- a/apps/papra-server/src/index.ts+++ b/apps/papra-server/src/index.ts@@ -30,6 +30,7 @@ const server = serve(if (config.ingestionFolder.isEnabled) {const { startWatchingIngestionFolders } = createIngestionFolderWatcher({+ taskServices,config,db,});diff --git a/apps/papra-server/src/modules/api-keys/e2e/create-document-with-api-key.e2e.test.ts b/apps/papra-server/src/modules/api-keys/e2e/create-document-with-api-key.e2e.test.tsindex e44bdc39..c73571fe 100644--- a/apps/papra-server/src/modules/api-keys/e2e/create-document-with-api-key.e2e.test.ts+++ b/apps/papra-server/src/modules/api-keys/e2e/create-document-with-api-key.e2e.test.ts@@ -66,7 +66,6 @@ describe('api-key e2e', () => {originalSha256Hash: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',name: 'invoice.txt',mimeType: 'text/plain',- content: 'test',});const fetchDocumentResponse = await app.request(`/api/organizations/org_222222222222222222222222/documents/${document.id}`, {diff --git a/apps/papra-server/src/modules/documents/documents.routes.ts b/apps/papra-server/src/modules/documents/documents.routes.tsindex 457ab797..8df37172 100644--- a/apps/papra-server/src/modules/documents/documents.routes.ts+++ b/apps/papra-server/src/modules/documents/documents.routes.ts@@ -35,7 +35,7 @@ export function registerDocumentsRoutes(context: RouteDefinitionContext) {setupUpdateDocumentRoute(context);}-function setupCreateDocumentRoute({ app, config, db, trackingServices }: RouteDefinitionContext) {+function setupCreateDocumentRoute({ app, config, db, trackingServices, taskServices }: RouteDefinitionContext) {app.post('/api/organizations/:organizationId/documents',requireAuthentication({ apiKeyPermissions: ['documents:create'] }),@@ -93,6 +93,7 @@ function setupCreateDocumentRoute({ app, config, db, trackingServices }: RouteDeconst createDocument = await createDocumentCreationUsecase({db,config,+ taskServices,trackingServices,ocrLanguages,});diff --git a/apps/papra-server/src/modules/documents/documents.services.ts b/apps/papra-server/src/modules/documents/documents.services.tsindex 2da1835e..841fee05 100644--- a/apps/papra-server/src/modules/documents/documents.services.ts+++ b/apps/papra-server/src/modules/documents/documents.services.ts@@ -1,3 +1,7 @@+import type { Logger } from '@crowlog/logger';+import { extractTextFromFile } from '@papra/lecture';+import { createLogger } from '../shared/logger/logger';+export async function getFileSha256Hash({ file }: { file: File }) {const arrayBuffer = await file.arrayBuffer();const hash = await crypto.subtle.digest('SHA-256', arrayBuffer);@@ -9,3 +13,23 @@ export async function getFileSha256Hash({ file }: { file: File }) {hash: hashHex,};}++export async function extractDocumentText({+ file,+ ocrLanguages,+ logger = createLogger({ namespace: 'documents:services' }),+}: {+ file: File;+ ocrLanguages?: string[];+ logger?: Logger;+}) {+ const { textContent, error, extractorName } = await extractTextFromFile({ file, config: { tesseract: { languages: ocrLanguages } } });++ if (error) {+ logger.error({ error, extractorName }, 'Error while extracting text from document');+ }++ return {+ text: textContent ?? '',+ };+}diff --git a/apps/papra-server/src/modules/documents/documents.usecases.test.ts b/apps/papra-server/src/modules/documents/documents.usecases.test.tsindex 80e51a4b..f3c034be 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.test.ts@@ -5,16 +5,18 @@ import { ORGANIZATION_ROLES } from '../organizations/organizations.constants';import { nextTick } from '../shared/async/defer.test-utils';import { collectReadableStreamToString } from '../shared/streams/readable-stream';import { documentsTagsTable } from '../tags/tags.table';+import { createInMemoryTaskServices } from '../tasks/tasks.test-utils';import { documentActivityLogTable } from './document-activity/document-activity.table';import { createDocumentAlreadyExistsError } from './documents.errors';import { createDocumentsRepository } from './documents.repository';import { documentsTable } from './documents.table';-import { createDocumentCreationUsecase } from './documents.usecases';+import { createDocumentCreationUsecase, extractAndSaveDocumentFileContent } from './documents.usecases';import { createDocumentStorageService } from './storage/documents.storage.services';describe('documents usecases', () => {describe('createDocument', () => {test('creating a document save the file to the storage and registers a record in the db', async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({users: [{ id: 'user-1', email: '[email protected]' }],organizations: [{ id: 'organization-1', name: 'Organization 1' }],@@ -32,6 +34,7 @@ describe('documents usecases', () => {config,generateDocumentId: () => 'doc_1',documentsStorageService,+ taskServices,});const file = new File(['content'], 'file.txt', { type: 'text/plain' });@@ -70,6 +73,7 @@ describe('documents usecases', () => {});test('in the same organization, we should not be able to have two documents with the same content, an error is raised if the document already exists', async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({users: [{ id: 'user-1', email: '[email protected]' }],organizations: [{ id: 'organization-1', name: 'Organization 1' }],@@ -89,6 +93,7 @@ describe('documents usecases', () => {config,generateDocumentId: () => `doc_${documentIdIndex++}`,documentsStorageService,+ taskServices,});const file = new File(['content'], 'file.txt', { type: 'text/plain' });@@ -178,6 +183,8 @@ describe('documents usecases', () => {],});+ const taskServices = createInMemoryTaskServices();+const config = overrideConfig({organizationPlans: { isFreePlanUnlimited: true },documentsStorage: { driver: 'in-memory' },@@ -186,6 +193,7 @@ describe('documents usecases', () => {const createDocument = await createDocumentCreationUsecase({db,config,+ taskServices,});// 3. Re-create the document@@ -219,6 +227,7 @@ describe('documents usecases', () => {});test('when there is an issue when inserting the document in the db, the file should not be saved in the storage', async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({users: [{ id: 'user-1', email: '[email protected]' }],organizations: [{ id: 'organization-1', name: 'Organization 1' }],@@ -243,6 +252,7 @@ describe('documents usecases', () => {throw new Error('Macron, explosion!');},},+ taskServices,});const file = new File(['content'], 'file.txt', { type: 'text/plain' });@@ -267,6 +277,7 @@ describe('documents usecases', () => {});test('when a document is created by a user, a document activity log is registered with the user id', async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({users: [{ id: 'user-1', email: '[email protected]' }],organizations: [{ id: 'organization-1', name: 'Organization 1' }],@@ -283,6 +294,7 @@ describe('documents usecases', () => {db,config,generateDocumentId: () => `doc_${documentIdIndex++}`,+ taskServices,});await createDocument({@@ -317,4 +329,53 @@ describe('documents usecases', () => {});});});++ describe('extractAndSaveDocumentFileContent', () => {+ test('given a stored document, its content is extracted and saved in the db', async () => {+ const { db } = await createInMemoryDatabase({+ users: [{ id: 'user-1', email: '[email protected]' }],+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],+ });++ const config = overrideConfig({+ organizationPlans: { isFreePlanUnlimited: true },+ documentsStorage: { driver: 'in-memory' },+ });++ const documentsRepository = createDocumentsRepository({ db });+ const documentsStorageService = await createDocumentStorageService({ config });++ await db.insert(documentsTable).values({+ id: 'document-1',+ organizationId: 'organization-1',+ originalStorageKey: 'organization-1/originals/document-1.txt',+ mimeType: 'text/plain',+ name: 'file-1.txt',+ originalName: 'file-1.txt',+ originalSha256Hash: 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9',+ });++ await documentsStorageService.saveFile({+ file: new File(['hello world'], 'file-1.txt', { type: 'text/plain' }),+ storageKey: 'organization-1/originals/document-1.txt',+ });++ await extractAndSaveDocumentFileContent({+ documentId: 'document-1',+ organizationId: 'organization-1',+ documentsRepository,+ documentsStorageService,+ });++ const documentRecords = await db.select().from(documentsTable);++ expect(documentRecords.length).to.eql(1);+ expect(documentRecords[0]).to.deep.include({+ id: 'document-1',+ organizationId: 'organization-1',+ content: 'hello world', // The content is extracted and saved in the db+ });+ });+ });});diff --git a/apps/papra-server/src/modules/documents/documents.usecases.ts b/apps/papra-server/src/modules/documents/documents.usecases.tsindex 03827980..9d2555d1 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.ts@@ -5,6 +5,7 @@ import type { Logger } from '../shared/logger/logger';import type { SubscriptionsRepository } from '../subscriptions/subscriptions.repository';import type { TaggingRulesRepository } from '../tagging-rules/tagging-rules.repository';import type { TagsRepository } from '../tags/tags.repository';+import type { TaskServices } from '../tasks/tasks.services';import type { TrackingServices } from '../tracking/tracking.services';import type { WebhookRepository } from '../webhooks/webhook.repository';import type { DocumentActivityRepository } from './document-activity/document-activity.repository';@@ -12,11 +13,11 @@ import type { DocumentsRepository } from './documents.repository';import type { Document } from './documents.types';import type { DocumentStorageService } from './storage/documents.storage.services';import { safely } from '@corentinth/chisels';-import { extractTextFromFile } from '@papra/lecture';import pLimit from 'p-limit';import { checkIfOrganizationCanCreateNewDocument } from '../organizations/organizations.usecases';import { createPlansRepository } from '../plans/plans.repository';import { createLogger } from '../shared/logger/logger';+import { collectStreamToFile } from '../shared/streams/stream.convertion';import { isDefined } from '../shared/utils';import { createSubscriptionsRepository } from '../subscriptions/subscriptions.repository';import { createTaggingRulesRepository } from '../tagging-rules/tagging-rules.repository';@@ -30,23 +31,9 @@ import { deferRegisterDocumentActivityLog } from './document-activity/document-aimport { createDocumentAlreadyExistsError, createDocumentNotDeletedError, createDocumentNotFoundError } from './documents.errors';import { buildOriginalDocumentKey, generateDocumentId as generateDocumentIdImpl } from './documents.models';import { createDocumentsRepository } from './documents.repository';-import { getFileSha256Hash } from './documents.services';+import { extractDocumentText, getFileSha256Hash } from './documents.services';import { createDocumentStorageService } from './storage/documents.storage.services';-const logger = createLogger({ namespace: 'documents:usecases' });--export async function extractDocumentText({ file, ocrLanguages }: { file: File; ocrLanguages?: string[] }) {- const { textContent, error, extractorName } = await extractTextFromFile({ file, config: { tesseract: { languages: ocrLanguages } } });-- if (error) {- logger.error({ error, extractorName }, 'Error while extracting text from document');- }-- return {- text: textContent ?? '',- };-}-export async function createDocument({file,userId,@@ -62,6 +49,7 @@ export async function createDocument({tagsRepository,webhookRepository,documentActivityRepository,+ taskServices,logger = createLogger({ namespace: 'documents:usecases' }),}: {file: File;@@ -78,6 +66,7 @@ export async function createDocument({tagsRepository: TagsRepository;webhookRepository: WebhookRepository;documentActivityRepository: DocumentActivityRepository;+ taskServices: TaskServices;logger?: Logger;}) {const {@@ -120,7 +109,6 @@ export async function createDocument({documentsStorageService,generateDocumentId,trackingServices,- ocrLanguages,logger,});@@ -146,6 +134,11 @@ export async function createDocument({},});+ await taskServices.scheduleJob({+ taskName: 'extract-document-file-content',+ data: { documentId: document.id, organizationId, ocrLanguages },+ });+return { document };}@@ -155,9 +148,11 @@ export type DocumentUsecaseDependencies = Omit<Parameters<typeof createDocument>export async function createDocumentCreationUsecase({db,config,+ taskServices,...initialDeps}: {db: Database;+ taskServices: TaskServices;config: Config;} & Partial<DocumentUsecaseDependencies>) {const deps = {@@ -176,7 +171,7 @@ export async function createDocumentCreationUsecase({logger: initialDeps.logger,};- return async (args: { file: File; userId?: string; organizationId: string }) => createDocument({ ...args, ...deps });+ return async (args: { file: File; userId?: string; organizationId: string }) => createDocument({ taskServices, ...args, ...deps });}async function handleExistingDocument({@@ -222,7 +217,6 @@ async function createNewDocument({documentsStorageService,generateDocumentId,trackingServices,- ocrLanguages,logger,}: {file: File;@@ -236,7 +230,6 @@ async function createNewDocument({documentsStorageService: DocumentStorageService;generateDocumentId: () => string;trackingServices: TrackingServices;- ocrLanguages?: string[];logger: Logger;}) {const documentId = generateDocumentId();@@ -252,8 +245,6 @@ async function createNewDocument({storageKey: originalDocumentStorageKey,});- const { text } = await extractDocumentText({ file, ocrLanguages });-const [result, error] = await safely(documentsRepository.saveOrganizationDocument({id: documentId,name: fileName,@@ -263,7 +254,6 @@ async function createNewDocument({originalSize: size,originalStorageKey: storageKey,mimeType,- content: text,originalSha256Hash: hash,}));@@ -412,3 +402,31 @@ export async function deleteAllTrashDocuments({documents.map(async document => limit(async () => hardDeleteDocument({ document, documentsRepository, documentsStorageService }))),);}++export async function extractAndSaveDocumentFileContent({+ documentId,+ organizationId,+ documentsRepository,+ documentsStorageService,+ ocrLanguages,+}: {+ documentId: string;+ ocrLanguages?: string[];+ organizationId: string;+ documentsRepository: DocumentsRepository;+ documentsStorageService: DocumentStorageService;+}) {+ const { document } = await documentsRepository.getDocumentById({ documentId, organizationId });++ if (!document) {+ throw createDocumentNotFoundError();+ }++ const { fileStream } = await documentsStorageService.getFileStream({ storageKey: document.originalStorageKey });++ const { file } = await collectStreamToFile({ fileStream, fileName: document.name, mimeType: document.mimeType });++ const { text } = await extractDocumentText({ file, ocrLanguages });++ await documentsRepository.updateDocument({ documentId, organizationId, content: text });+}diff --git a/apps/papra-server/src/modules/documents/tasks/extract-document-file-content.task.ts b/apps/papra-server/src/modules/documents/tasks/extract-document-file-content.task.tsnew file mode 100644index 00000000..87672f9b--- /dev/null+++ b/apps/papra-server/src/modules/documents/tasks/extract-document-file-content.task.ts@@ -0,0 +1,29 @@+import type { Database } from '../../app/database/database.types';+import type { Config } from '../../config/config.types';+import type { TaskServices } from '../../tasks/tasks.services';+import { createDocumentsRepository } from '../documents.repository';+import { extractAndSaveDocumentFileContent } from '../documents.usecases';+import { createDocumentStorageService } from '../storage/documents.storage.services';++export async function registerExtractDocumentFileContentTask({ taskServices, db, config }: { taskServices: TaskServices; db: Database; config: Config }) {+ const taskName = 'extract-document-file-content';++ taskServices.registerTask({+ taskName,+ handler: async ({ data }) => {+ const documentsRepository = createDocumentsRepository({ db });+ const documentsStorageService = await createDocumentStorageService({ config });++ // TODO: remove type cast+ const { documentId, organizationId, ocrLanguages } = data as { documentId: string; organizationId: string; ocrLanguages: string[] };++ await extractAndSaveDocumentFileContent({+ documentId,+ organizationId,+ ocrLanguages,+ documentsRepository,+ documentsStorageService,+ });+ },+ });+}diff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.tsindex 788bdec3..01ab9ffa 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.test.ts@@ -11,6 +11,7 @@ import { createOrganizationsRepository } from '../organizations/organizations.reimport { createInMemoryFsServices } from '../shared/fs/fs.in-memory';import { createFsServices } from '../shared/fs/fs.services';import { createTestLogger } from '../shared/logger/logger.test-utils';+import { createInMemoryTaskServices } from '../tasks/tasks.test-utils';import { createInvalidPostProcessingStrategyError } from './ingestion-folders.errors';import { moveIngestionFile, processFile } from './ingestion-folders.usecases';@@ -18,6 +19,7 @@ describe('ingestion-folders usecases', () => {describe('processFile', () => {describe('when a file is added to an organization ingestion folder', () => {test('if the post processing strategy is set to "move", the file is ingested and moved to the done folder', async () => {+ const taskServices = createInMemoryTaskServices();const { logger, getLogs } = createTestLogger();const { db } = await createInMemoryDatabase({@@ -53,7 +55,7 @@ describe('ingestion-folders usecases', () => {organizationsRepository,logger,fs: createFsServices({ fs: vol.promises as unknown as FsNative }),- createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId }),+ createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});// Check database@@ -120,6 +122,7 @@ describe('ingestion-folders usecases', () => {});test('if the post processing strategy is set to "delete", the file is ingested and deleted', async () => {+ const taskServices = createInMemoryTaskServices();const { logger, getLogs } = createTestLogger();const { db } = await createInMemoryDatabase({@@ -154,7 +157,7 @@ describe('ingestion-folders usecases', () => {organizationsRepository,logger,fs: createFsServices({ fs: vol.promises as unknown as FsNative }),- createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId }),+ createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});// Check database@@ -221,6 +224,7 @@ describe('ingestion-folders usecases', () => {});test('if the post processing strategy is not implemented, an error is thrown after the file has been ingested', async () => {+ const taskServices = createInMemoryTaskServices();const { logger } = createTestLogger();const { db } = await createInMemoryDatabase({@@ -256,7 +260,7 @@ describe('ingestion-folders usecases', () => {organizationsRepository,logger,fs: createFsServices({ fs: vol.promises as unknown as FsNative }),- createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId }),+ createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),}));expect(error).to.deep.equal(createInvalidPostProcessingStrategyError({ strategy: 'unknown' }));@@ -293,6 +297,7 @@ describe('ingestion-folders usecases', () => {});test('if for some reason the file cannot be read, a log is emitted and the processing stops', async () => {+ const taskServices = createInMemoryTaskServices();const { logger, getLogs } = createTestLogger();const { db } = await createInMemoryDatabase({@@ -333,7 +338,7 @@ describe('ingestion-folders usecases', () => {throw new Error('File not found');},},- createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId }),+ createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});// Check logs@@ -433,6 +438,7 @@ describe('ingestion-folders usecases', () => {});test('when the document already exists in the database, it is not ingested, but the post-processing is still executed', async () => {+ const taskServices = createInMemoryTaskServices();const { logger, getLogs } = createTestLogger();// This is the sha256 hash of the "lorem ipsum" text@@ -472,7 +478,7 @@ describe('ingestion-folders usecases', () => {organizationsRepository,logger,fs: createFsServices({ fs: vol.promises as unknown as FsNative }),- createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId }),+ createDocument: await createDocumentCreationUsecase({ db, config, logger, documentsStorageService, generateDocumentId, taskServices }),});// Check databasediff --git a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.tsindex d2677dab..aac31e3e 100644--- a/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts+++ b/apps/papra-server/src/modules/ingestion-folders/ingestion-folders.usecases.ts@@ -5,6 +5,7 @@ import type { CreateDocumentUsecase } from '../documents/documents.usecases';import type { OrganizationsRepository } from '../organizations/organizations.repository';import type { FsServices } from '../shared/fs/fs.services';import type { Logger } from '../shared/logger/logger';+import type { TaskServices } from '../tasks/tasks.services';import { isAbsolute, join, parse } from 'node:path';import { safely } from '@corentinth/chisels';import chokidar from 'chokidar';@@ -27,10 +28,12 @@ export function createIngestionFolderWatcher({config,logger = createLogger({ namespace: 'ingestion-folder-watcher' }),db,+ taskServices,}: {config: Config;logger?: Logger;db: Database;+ taskServices: TaskServices;}) {const { folderRootPath, watcher: { usePolling, pollingInterval }, processingConcurrency } = config.ingestionFolder;@@ -41,7 +44,7 @@ export function createIngestionFolderWatcher({return {startWatchingIngestionFolders: async () => {const organizationsRepository = createOrganizationsRepository({ db });- const createDocument = await createDocumentCreationUsecase({ db, config, logger });+ const createDocument = await createDocumentCreationUsecase({ db, config, logger, taskServices });const ignored = await buildPathIgnoreFunction({ config, cwd, organizationsRepository });diff --git a/apps/papra-server/src/modules/intake-emails/e2e/ingest-intake-emails.e2e.test.ts b/apps/papra-server/src/modules/intake-emails/e2e/ingest-intake-emails.e2e.test.tsindex ed6e3aa5..ac84cde5 100644--- a/apps/papra-server/src/modules/intake-emails/e2e/ingest-intake-emails.e2e.test.ts+++ b/apps/papra-server/src/modules/intake-emails/e2e/ingest-intake-emails.e2e.test.ts@@ -154,14 +154,13 @@ describe('intake-emails e2e', () => {const [document] = documents;expect(- pick(document, ['organizationId', 'createdBy', 'mimeType', 'originalName', 'originalSize', 'content']),+ pick(document, ['organizationId', 'createdBy', 'mimeType', 'originalName', 'originalSize']),).to.eql({organizationId: 'org_1',createdBy: null,mimeType: 'text/plain',originalName: 'test.txt',originalSize: 11,- content: 'hello world',});});});diff --git a/apps/papra-server/src/modules/intake-emails/intake-emails.routes.ts b/apps/papra-server/src/modules/intake-emails/intake-emails.routes.tsindex d9dbe7fa..28a1662a 100644--- a/apps/papra-server/src/modules/intake-emails/intake-emails.routes.ts+++ b/apps/papra-server/src/modules/intake-emails/intake-emails.routes.ts@@ -145,7 +145,7 @@ function setupUpdateIntakeEmailRoute({ app, db }: RouteDefinitionContext) {);}-function setupIngestIntakeEmailRoute({ app, db, config, trackingServices }: RouteDefinitionContext) {+function setupIngestIntakeEmailRoute({ app, db, config, trackingServices, taskServices }: RouteDefinitionContext) {app.post(INTAKE_EMAILS_INGEST_ROUTE,validateFormData(z.object({@@ -193,6 +193,7 @@ function setupIngestIntakeEmailRoute({ app, db, config, trackingServices }: Routdb,config,trackingServices,+ taskServices,});await processIntakeEmailIngestion({diff --git a/apps/papra-server/src/modules/intake-emails/intake-emails.usecases.test.ts b/apps/papra-server/src/modules/intake-emails/intake-emails.usecases.test.tsindex 1eabf67c..53436b0d 100644--- a/apps/papra-server/src/modules/intake-emails/intake-emails.usecases.test.ts+++ b/apps/papra-server/src/modules/intake-emails/intake-emails.usecases.test.ts@@ -10,6 +10,7 @@ import { createDocumentCreationUsecase } from '../documents/documents.usecases';import { PLUS_PLAN_ID } from '../plans/plans.constants';import { createLogger } from '../shared/logger/logger';import { createSubscriptionsRepository } from '../subscriptions/subscriptions.repository';+import { createInMemoryTaskServices } from '../tasks/tasks.test-utils';import { createIntakeEmailLimitReachedError } from './intake-emails.errors';import { createIntakeEmailsRepository } from './intake-emails.repository';import { intakeEmailsTable } from './intake-emails.tables';@@ -19,6 +20,7 @@ describe('intake-emails usecases', () => {describe('ingestEmailForRecipient', () => {describe('when a email is forwarded to papra api, we look for the recipient in the intake emails repository and create a papra document for each attachment', () => {test(`when an intake email is is configured, enabled and match the recipient, and the sender is allowed, a document is created for each attachment`, async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({organizations: [{ id: 'org-1', name: 'Organization 1' }],intakeEmails: [{ id: 'ie-1', organizationId: 'org-1', allowedOrigins: ['[email protected]'], emailAddress: '[email protected]' }],@@ -28,6 +30,7 @@ describe('intake-emails usecases', () => {const createDocument = await createDocumentCreationUsecase({db,+ taskServices,config: overrideConfig({documentsStorage: { driver: 'in-memory' },organizationPlans: { isFreePlanUnlimited: true },@@ -48,10 +51,10 @@ describe('intake-emails usecases', () => {const documents = await db.select().from(documentsTable).orderBy(asc(documentsTable.name));expect(- documents.map(doc => pick(doc, ['organizationId', 'name', 'mimeType', 'originalName', 'content'])),+ documents.map(doc => pick(doc, ['organizationId', 'name', 'mimeType', 'originalName'])),).to.eql([- { organizationId: 'org-1', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt', content: 'content1' },- { organizationId: 'org-1', name: 'file2.txt', mimeType: 'text/plain', originalName: 'file2.txt', content: 'content2' },+ { organizationId: 'org-1', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt' },+ { organizationId: 'org-1', name: 'file2.txt', mimeType: 'text/plain', originalName: 'file2.txt' },]);});@@ -59,6 +62,7 @@ describe('intake-emails usecases', () => {const loggerTransport = createInMemoryLoggerTransport();const logger = createLogger({ transports: [loggerTransport], namespace: 'test' });+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({organizations: [{ id: 'org-1', name: 'Organization 1' }],intakeEmails: [{ id: 'ie-1', organizationId: 'org-1', isEnabled: false, emailAddress: '[email protected]' }],@@ -68,6 +72,7 @@ describe('intake-emails usecases', () => {const createDocument = await createDocumentCreationUsecase({db,+ taskServices,config: overrideConfig({documentsStorage: { driver: 'in-memory' },organizationPlans: { isFreePlanUnlimited: true },@@ -90,6 +95,7 @@ describe('intake-emails usecases', () => {});test('when no intake email is found for the recipient, nothing happens, only a log is emitted', async () => {+ const taskServices = createInMemoryTaskServices();const loggerTransport = createInMemoryLoggerTransport();const logger = createLogger({ transports: [loggerTransport], namespace: 'test' });@@ -99,6 +105,7 @@ describe('intake-emails usecases', () => {const createDocument = await createDocumentCreationUsecase({db,+ taskServices,config: overrideConfig({documentsStorage: { driver: 'in-memory' },organizationPlans: { isFreePlanUnlimited: true },@@ -123,6 +130,7 @@ describe('intake-emails usecases', () => {test(`in order to be processed, the emitter of the email must be allowed for the intake emailit should be registered in the intake email allowed originsif not, an error is logged and no document is created`, async () => {+ const taskServices = createInMemoryTaskServices();const loggerTransport = createInMemoryLoggerTransport();const logger = createLogger({ transports: [loggerTransport], namespace: 'test' });@@ -135,6 +143,7 @@ describe('intake-emails usecases', () => {const createDocument = await createDocumentCreationUsecase({db,+ taskServices,config: overrideConfig({documentsStorage: { driver: 'in-memory' },organizationPlans: { isFreePlanUnlimited: true },@@ -167,6 +176,7 @@ describe('intake-emails usecases', () => {describe('processIntakeEmailIngestion', () => {test(`when an email is send to multiple intake emails from different organization, the attachments are processed for each of them`, async () => {+ const taskServices = createInMemoryTaskServices();const { db } = await createInMemoryDatabase({organizations: [{ id: 'org-1', name: 'Organization 1' },@@ -182,6 +192,7 @@ describe('intake-emails usecases', () => {const createDocument = await createDocumentCreationUsecase({db,+ taskServices,config: overrideConfig({documentsStorage: { driver: 'in-memory' },organizationPlans: { isFreePlanUnlimited: true },@@ -201,10 +212,10 @@ describe('intake-emails usecases', () => {const documents = await db.select().from(documentsTable).orderBy(asc(documentsTable.organizationId));expect(- documents.map(doc => pick(doc, ['organizationId', 'name', 'mimeType', 'originalName', 'content'])),+ documents.map(doc => pick(doc, ['organizationId', 'name', 'mimeType', 'originalName'])),).to.eql([- { organizationId: 'org-1', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt', content: 'content1' },- { organizationId: 'org-2', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt', content: 'content1' },+ { organizationId: 'org-1', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt' },+ { organizationId: 'org-2', name: 'file1.txt', mimeType: 'text/plain', originalName: 'file1.txt' },]);});});diff --git a/apps/papra-server/src/modules/shared/streams/stream.convertion.ts b/apps/papra-server/src/modules/shared/streams/stream.convertion.tsnew file mode 100644index 00000000..f4490293--- /dev/null+++ b/apps/papra-server/src/modules/shared/streams/stream.convertion.ts@@ -0,0 +1,8 @@+export async function collectStreamToFile({ fileStream, fileName, mimeType }: { fileStream: ReadableStream; fileName: string; mimeType: string }): Promise<{ file: File }> {+ const response = new Response(fileStream);+ const blob = await response.blob();++ const file = new File([blob], fileName, { type: mimeType });++ return { file };+}diff --git a/apps/papra-server/src/modules/tasks/tasks.definitions.ts b/apps/papra-server/src/modules/tasks/tasks.definitions.tsindex aa21b0ba..b800b337 100644--- a/apps/papra-server/src/modules/tasks/tasks.definitions.ts+++ b/apps/papra-server/src/modules/tasks/tasks.definitions.ts@@ -1,10 +1,12 @@import type { Database } from '../app/database/database.types';import type { Config } from '../config/config.types';import type { TaskServices } from './tasks.services';+import { registerExtractDocumentFileContentTask } from '../documents/tasks/extract-document-file-content.task';import { registerHardDeleteExpiredDocumentsTask } from '../documents/tasks/hard-delete-expired-documents.task';import { registerExpireInvitationsTask } from '../organizations/tasks/expire-invitations.task';export async function registerTaskDefinitions({ taskServices, db, config }: { taskServices: TaskServices; db: Database; config: Config }) {await registerHardDeleteExpiredDocumentsTask({ taskServices, db, config });await registerExpireInvitationsTask({ taskServices, db, config });+ await registerExtractDocumentFileContentTask({ taskServices, db, config });}diff --git a/apps/papra-server/src/modules/tasks/tasks.services.ts b/apps/papra-server/src/modules/tasks/tasks.services.tsindex 0aed081f..ea9b3b12 100644--- a/apps/papra-server/src/modules/tasks/tasks.services.ts+++ b/apps/papra-server/src/modules/tasks/tasks.services.ts@@ -11,7 +11,7 @@ export function createTaskServices({ config }: { config: Config }) {const workerId = config.tasks.worker.id ?? 'default';const driver = createMemoryDriver();- const cadence = createCadence({ driver });+ const cadence = createCadence({ driver, logger });return {...cadence,diff --git a/apps/papra-server/src/modules/tasks/tasks.test-utils.ts b/apps/papra-server/src/modules/tasks/tasks.test-utils.tsnew file mode 100644index 00000000..b1f05408--- /dev/null+++ b/apps/papra-server/src/modules/tasks/tasks.test-utils.ts@@ -0,0 +1,8 @@+import { overrideConfig } from '../config/config.test-utils';+import { createTaskServices } from './tasks.services';++export function createInMemoryTaskServices({ workerId = 'test' }: { workerId?: string } = {}) {+ const config = overrideConfig({ tasks: { worker: { id: workerId } } });++ return createTaskServices({ config });+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 3e97c107..f94ae97d 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -251,11 +251,11 @@ importers:specifier: ^12.27.0version: 12.27.0'@cadence-mq/core':- specifier: ^0.1.0- version: 0.1.0+ specifier: ^0.2.1+ version: 0.2.1'@cadence-mq/driver-memory':- specifier: ^0.1.0- version: 0.1.0(@cadence-mq/[email protected])+ specifier: ^0.2.0+ version: 0.2.0(@cadence-mq/[email protected])'@corentinth/chisels':specifier: ^1.3.1version: 1.3.1@@ -1050,13 +1050,13 @@ packages:'@better-fetch/[email protected]':resolution: {integrity: sha512-rEFOE1MYIsBmoMJtQbl32PGHHXuG2hDxvEd7rUHE0vCBoFQVSDqaVs9hkZEtHCxRoY+CljXKFCOuJ8uxqw1LcA==}- '@cadence-mq/[email protected]':- resolution: {integrity: sha512-bJjYI3brZLQqV3kxz4/cQs2C1dhBL1j6PEt51nYNobSxsanyRNF7/u/Zdlcj0uMC2Pe0uepryrM78/N0r3s4ZQ==}+ '@cadence-mq/[email protected]':+ resolution: {integrity: sha512-Cu/jqR7mNhMZ1U4Boiudy2nePyf4PtqBUFGhUcsCQPJfymKcrDm4xjp8A/2tKZr5JSgkN/7L0/+mHZ27GVSryQ==}- '@cadence-mq/[email protected]':- resolution: {integrity: sha512-wqG+4+bFrXNo0bCsdMtc0U6oxM8dWFG3nGRmKobwdTcnQ42z93Xcm7v1m3kxQibJAi5fjSrS0z6oTIkatorvAg==}+ '@cadence-mq/[email protected]':+ resolution: {integrity: sha512-U/L5nkCu+BYO814oQAYbFYSNASha+6Om3Px3+Jm47YzFmSrQhrnX6fTyICVQFz+MflWndhke6Bh1mwik6nbrcw==}peerDependencies:- '@cadence-mq/core': ^0.1.0+ '@cadence-mq/core': ^0.2.0'@capsizecss/[email protected]':resolution: {integrity: sha512-GrSU71meACqcmIUxPYOJvGKF0yryjN/L1aCuE9DViCTJI7bfkjgYDPD1zbNDcINJwSSP6UaBZY9GAbYDO7re0Q==}@@ -9395,16 +9395,16 @@ snapshots:'@better-fetch/[email protected]': {}- '@cadence-mq/[email protected]':+ '@cadence-mq/[email protected]':dependencies:'@corentinth/chisels': 1.3.1'@paralleldrive/cuid2': 2.2.2'@standard-schema/spec': 1.0.0cron-parser: 5.3.0- '@cadence-mq/[email protected](@cadence-mq/[email protected])':+ '@cadence-mq/[email protected](@cadence-mq/[email protected])':dependencies:- '@cadence-mq/core': 0.1.0+ '@cadence-mq/core': 0.2.1'@capsizecss/[email protected]':dependencies: