The minimalistic document archiving platform.
refactor(documents): narrowed down the document storage config parameter (#484)
Corentin Thomasset9730a06468ff
diff --git a/apps/papra-server/src/index.ts b/apps/papra-server/src/index.tsindex 4fd496fb..1fc8494c 100644--- a/apps/papra-server/src/index.ts+++ b/apps/papra-server/src/index.ts@@ -18,7 +18,7 @@ const { config } = await parseConfig({ env });await ensureLocalDatabaseDirectoryExists({ config });const { db, client } = setupDatabase(config.database);-const documentsStorageService = createDocumentStorageService({ config });+const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });const taskServices = createTaskServices({ config });const { app } = await createServer({ config, db, taskServices, documentsStorageService });diff --git a/apps/papra-server/src/modules/app/server.ts b/apps/papra-server/src/modules/app/server.tsindex 13ca02c4..31048f31 100644--- a/apps/papra-server/src/modules/app/server.ts+++ b/apps/papra-server/src/modules/app/server.ts@@ -26,7 +26,7 @@ async function createGlobalDependencies(partialDeps: Partial<GlobalDependencies>const auth = partialDeps.auth ?? getAuth({ db, config, authEmailsServices: createAuthEmailsServices({ emailsServices }), trackingServices }).auth;const subscriptionsServices = createSubscriptionsServices({ config });const taskServices = partialDeps.taskServices ?? createTaskServices({ config });- const documentsStorageService = partialDeps.documentsStorageService ?? createDocumentStorageService({ config });+ const documentsStorageService = partialDeps.documentsStorageService ?? createDocumentStorageService({ documentStorageConfig: config.documentsStorage });return {documentsStorageService,diff --git a/apps/papra-server/src/modules/documents/documents.usecases.test.ts b/apps/papra-server/src/modules/documents/documents.usecases.test.tsindex 913a6f77..2fdbe1f5 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.test.ts@@ -33,7 +33,7 @@ describe('documents usecases', () => {organizationPlans: { isFreePlanUnlimited: true },documentsStorage: { driver: 'in-memory' },});- const documentsStorageService = createDocumentStorageService({ config });+ const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });const createDocument = await createDocumentCreationUsecase({db,@@ -93,7 +93,7 @@ describe('documents usecases', () => {documentsStorage: { driver: 'in-memory' },});- const documentsStorageService = createDocumentStorageService({ config });+ const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });let documentIdIndex = 1;const createDocument = await createDocumentCreationUsecase({@@ -254,7 +254,7 @@ describe('documents usecases', () => {});const documentsRepository = createDocumentsRepository({ db });- const documentsStorageService = createDocumentStorageService({ config });+ const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });const createDocument = await createDocumentCreationUsecase({documentsStorageService,@@ -533,7 +533,7 @@ describe('documents usecases', () => {});const documentsRepository = createDocumentsRepository({ db });- const documentsStorageService = createDocumentStorageService({ config });+ const documentsStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });const taggingRulesRepository = createTaggingRulesRepository({ db });const tagsRepository = createTagsRepository({ db });diff --git a/apps/papra-server/src/modules/documents/storage/documents.storage.services.ts b/apps/papra-server/src/modules/documents/storage/documents.storage.services.tsindex cb921e99..93adcb30 100644--- a/apps/papra-server/src/modules/documents/storage/documents.storage.services.ts+++ b/apps/papra-server/src/modules/documents/storage/documents.storage.services.ts@@ -1,4 +1,5 @@import type { Config } from '../../config/config.types';+import type { DocumentStorageConfig } from './documents.storage.types';import type { StorageDriver, StorageDriverFactory, StorageServices } from './drivers/drivers.models';import { createError } from '../../shared/errors/errors';import { isNil } from '../../shared/utils';@@ -17,8 +18,8 @@ const storageDriverFactories = {export type DocumentStorageService = Awaited<ReturnType<typeof createDocumentStorageService>>;-export function createDocumentStorageService({ config }: { config: Config }): StorageServices {- const storageDriverName = config.documentsStorage.driver;+export function createDocumentStorageService({ documentStorageConfig }: { documentStorageConfig: DocumentStorageConfig }): StorageServices {+ const storageDriverName = documentStorageConfig.driver;const storageDriverFactory: StorageDriverFactory | undefined = storageDriverFactories[storageDriverName];@@ -31,11 +32,11 @@ export function createDocumentStorageService({ config }: { config: Config }): St});}- const storageDriver = storageDriverFactory({ config });+ const storageDriver = storageDriverFactory({ documentStorageConfig });return createDocumentStorageServiceFromDriver({storageDriver,- encryptionConfig: config.documentsStorage.encryption,+ encryptionConfig: documentStorageConfig.encryption,});}diff --git a/apps/papra-server/src/modules/documents/storage/documents.storage.types.ts b/apps/papra-server/src/modules/documents/storage/documents.storage.types.tsnew file mode 100644index 00000000..7c2902b0--- /dev/null+++ b/apps/papra-server/src/modules/documents/storage/documents.storage.types.ts@@ -0,0 +1,3 @@+import type { Config } from '../../config/config.types';++export type DocumentStorageConfig = Config['documentsStorage'];diff --git a/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.int.test.ts b/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.int.test.tsindex a270a304..b40538c5 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.int.test.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.int.test.ts@@ -1,7 +1,7 @@+import type { DocumentStorageConfig } from '../../documents.storage.types';import { AzuriteContainer } from '@testcontainers/azurite';import { describe } from 'vitest';import { TEST_CONTAINER_IMAGES } from '../../../../../../test/containers/images';-import { overrideConfig } from '../../../../config/config.test-utils';import { runDriverTestSuites } from '../drivers.test-suite';import { azBlobStorageDriverFactory } from './az-blob.storage-driver';@@ -13,11 +13,7 @@ describe('az-blob storage-driver', () => {const azuriteContainer = await new AzuriteContainer(TEST_CONTAINER_IMAGES.AZURITE).withInMemoryPersistence().start();const connectionString = azuriteContainer.getConnectionString();- const config = overrideConfig({- documentsStorage: { drivers: { azureBlob: { connectionString, containerName: 'test-container' } } },- });-- const driver = azBlobStorageDriverFactory({ config });+ const driver = azBlobStorageDriverFactory({ documentStorageConfig: { drivers: { azureBlob: { connectionString, containerName: 'test-container' } } } as DocumentStorageConfig });const client = driver.getClient();await client.createContainer('test-container');diff --git a/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.ts b/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.tsindex 3ab97257..aec1933f 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/az-blob/az-blob.storage-driver.ts@@ -11,8 +11,8 @@ function isAzureBlobNotFoundError(error: Error): boolean {return ('statusCode' in error && error.statusCode === 404) || ('code' in error && error.code === 'BlobNotFound');}-export const azBlobStorageDriverFactory = defineStorageDriver(({ config }) => {- const { accountName, accountKey, containerName, connectionString } = config.documentsStorage.drivers.azureBlob;+export const azBlobStorageDriverFactory = defineStorageDriver(({ documentStorageConfig }) => {+ const { accountName, accountKey, containerName, connectionString } = documentStorageConfig.drivers.azureBlob;const blobServiceClient = connectionString !== undefined? BlobServiceClient.fromConnectionString(connectionString)diff --git a/apps/papra-server/src/modules/documents/storage/drivers/drivers.models.ts b/apps/papra-server/src/modules/documents/storage/drivers/drivers.models.tsindex b5dc6fa5..58a63d1d 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/drivers.models.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/drivers.models.ts@@ -1,6 +1,6 @@import type { Readable } from 'node:stream';-import type { Config } from '../../../config/config.types';import type { ExtendNamedArguments, ExtendReturnPromise } from '../../../shared/types';+import type { DocumentStorageConfig } from '../documents.storage.types';export type StorageDriver = {name: string;@@ -31,7 +31,7 @@ export type StorageServices = {deleteFile: StorageDriver['deleteFile'];};-export type StorageDriverFactory = (args: { config: Config }) => StorageDriver;+export type StorageDriverFactory = (args: { documentStorageConfig: DocumentStorageConfig }) => StorageDriver;export function defineStorageDriver<T extends StorageDriverFactory>(factory: T) {return factory;diff --git a/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.int.test.ts b/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.int.test.tsindex c103280a..ad27187a 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.int.test.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.int.test.ts@@ -1,4 +1,4 @@-import type { Config } from '../../../../config/config.types';+import type { DocumentStorageConfig } from '../../documents.storage.types';import fs from 'node:fs';import { tmpdir } from 'node:os';import path, { join } from 'node:path';@@ -28,18 +28,10 @@ describe('storage driver', () => {createDriver: async () => {const tmpDirectory = await createTmpDirectory();- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-return {- driver: fsStorageDriverFactory({ config }),+ driver: fsStorageDriverFactory({+ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig,+ }),[Symbol.asyncDispose]: async () => {await deleteTmpDirectory(tmpDirectory);},@@ -49,17 +41,7 @@ describe('storage driver', () => {describe('saveFile', () => {test('persists the file to the filesystem', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });const { storageKey } = await fsStorageDriver.saveFile({fileStream: createReadableStream({ content: 'lorem ipsum' }),@@ -77,17 +59,7 @@ describe('storage driver', () => {});test('an error is raised if the file already exists', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });await fsStorageDriver.saveFile({fileStream: createReadableStream({ content: 'lorem ipsum' }),@@ -109,17 +81,7 @@ describe('storage driver', () => {describe('getFileStream', () => {test('get a readable stream of a stored file', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });await fsStorageDriver.saveFile({fileStream: createReadableStream({ content: 'lorem ipsum' }),@@ -139,17 +101,7 @@ describe('storage driver', () => {});test('an error is raised if the file does not exist', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });await expect(fsStorageDriver.getFileStream({ storageKey: 'org_1/text-file.txt' })).rejects.toThrow(createFileNotFoundError());});@@ -157,17 +109,7 @@ describe('storage driver', () => {describe('deleteFile', () => {test('deletes a stored file', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });await fsStorageDriver.saveFile({fileStream: createReadableStream({ content: 'lorem ipsum' }),@@ -189,17 +131,7 @@ describe('storage driver', () => {});test('when the file does not exist, an error is raised', async () => {- const config = {- documentsStorage: {- drivers: {- filesystem: {- root: tmpDirectory,- },- },- },- } as Config;-- const fsStorageDriver = fsStorageDriverFactory({ config });+ const fsStorageDriver = fsStorageDriverFactory({ documentStorageConfig: { drivers: { filesystem: { root: tmpDirectory } } } as DocumentStorageConfig });await expect(fsStorageDriver.deleteFile({ storageKey: 'org_1/text-file.txt' })).rejects.toThrow(createFileNotFoundError());});diff --git a/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.ts b/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.tsindex 60d55de0..22c115f6 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.ts@@ -8,8 +8,8 @@ import { createFileAlreadyExistsError } from './fs.storage-driver.errors';export const FS_STORAGE_DRIVER_NAME = 'filesystem' as const;-export const fsStorageDriverFactory = defineStorageDriver(({ config }) => {- const { root } = config.documentsStorage.drivers.filesystem;+export const fsStorageDriverFactory = defineStorageDriver(({ documentStorageConfig }) => {+ const { root } = documentStorageConfig.drivers.filesystem;const getStoragePath = ({ storageKey }: { storageKey: string }) => ({ storagePath: join(root, storageKey) });diff --git a/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.int.test.ts b/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.int.test.tsindex 2ec1daab..adaa02cb 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.int.test.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.int.test.ts@@ -1,8 +1,8 @@+import type { DocumentStorageConfig } from '../../documents.storage.types';import { CreateBucketCommand } from '@aws-sdk/client-s3';import { LocalstackContainer } from '@testcontainers/localstack';import { describe } from 'vitest';import { TEST_CONTAINER_IMAGES } from '../../../../../../test/containers/images';-import { overrideConfig } from '../../../../config/config.test-utils';import { runDriverTestSuites } from '../drivers.test-suite';import { s3StorageDriverFactory } from './s3.storage-driver';@@ -16,8 +16,8 @@ describe('s3 storage-driver', () => {const localstackContainer = await new LocalstackContainer(TEST_CONTAINER_IMAGES.LOCALSTACK).start();const bucketName = 'test-bucket';- const config = overrideConfig({- documentsStorage: {+ const driver = s3StorageDriverFactory({+ documentStorageConfig: {drivers: {s3: {accessKeyId: 'test',@@ -28,11 +28,9 @@ describe('s3 storage-driver', () => {forcePathStyle: true,},},- },+ } as DocumentStorageConfig,});- const driver = s3StorageDriverFactory({ config });-const s3Client = driver.getClient();await s3Client.send(new CreateBucketCommand({ Bucket: bucketName }));diff --git a/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.ts b/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.tsindex 09757c04..b3327f95 100644--- a/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.ts+++ b/apps/papra-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.ts@@ -15,8 +15,8 @@ function isS3NotFoundError(error: Error) {|| ('Code' in error && typeof error.Code === 'string' && codes.includes(error.Code));}-export const s3StorageDriverFactory = defineStorageDriver(({ config }) => {- const { accessKeyId, secretAccessKey, bucketName, region, endpoint, forcePathStyle } = config.documentsStorage.drivers.s3;+export const s3StorageDriverFactory = defineStorageDriver(({ documentStorageConfig }) => {+ const { accessKeyId, secretAccessKey, bucketName, region, endpoint, forcePathStyle } = documentStorageConfig.drivers.s3;const s3Client = new S3Client({region,diff --git a/apps/papra-server/src/modules/documents/storage/encryption/document-encryption.usecases.ts b/apps/papra-server/src/modules/documents/storage/encryption/document-encryption.usecases.tsindex 060a6b2e..a5387ae4 100644--- a/apps/papra-server/src/modules/documents/storage/encryption/document-encryption.usecases.ts+++ b/apps/papra-server/src/modules/documents/storage/encryption/document-encryption.usecases.ts@@ -17,7 +17,12 @@ export async function encryptAllUnencryptedDocuments({deleteUnencryptedAfterEncryption?: boolean;}) {const documents = await db- .select({ id: documentsTable.id, originalStorageKey: documentsTable.originalStorageKey, fileName: documentsTable.originalName, mimeType: documentsTable.mimeType })+ .select({+ id: documentsTable.id,+ originalStorageKey: documentsTable.originalStorageKey,+ fileName: documentsTable.originalName,+ mimeType: documentsTable.mimeType,+ }).from(documentsTable).where(isNull(documentsTable.fileEncryptionKeyWrapped)).orderBy(documentsTable.id);@@ -34,15 +39,26 @@ export async function encryptAllUnencryptedDocuments({fileEncryptionKekVersion: null,});const newStorageKey = `${originalStorageKey}.enc`;- const { storageKey, ...encryptionFields } = await documentStorageService.saveFile({ fileStream, fileName, mimeType, storageKey: newStorageKey });+ const { storageKey, ...encryptionFields }+ = await documentStorageService.saveFile({+ fileStream,+ fileName,+ mimeType,+ storageKey: newStorageKey,+ });- await db.update(documentsTable).set({- ...encryptionFields,- originalStorageKey: storageKey,- }).where(eq(documentsTable.id, id));+ await db+ .update(documentsTable)+ .set({+ ...encryptionFields,+ originalStorageKey: storageKey,+ })+ .where(eq(documentsTable.id, id));if (deleteUnencryptedAfterEncryption) {- await documentStorageService.deleteFile({ storageKey: originalStorageKey });+ await documentStorageService.deleteFile({+ storageKey: originalStorageKey,+ });}}}diff --git a/apps/papra-server/src/scripts/encrypt-all-documents.script.ts b/apps/papra-server/src/scripts/encrypt-all-documents.script.tsindex 7f128110..7231f67e 100644--- a/apps/papra-server/src/scripts/encrypt-all-documents.script.ts+++ b/apps/papra-server/src/scripts/encrypt-all-documents.script.ts@@ -9,7 +9,7 @@ import { runScriptWithDb } from './commons/run-script';await runScriptWithDb({ scriptName: 'encrypt-all-documents' },async ({ db, config, logger, isDryRun }) => {- const documentStorageService = createDocumentStorageService({ config });+ const documentStorageService = createDocumentStorageService({ documentStorageConfig: config.documentsStorage });if (!config.documentsStorage.encryption.isEncryptionEnabled) {logger.error('Document encryption is not enabled, skipping');