The minimalistic document archiving platform.
import type { PartialBy } from '@corentinth/chisels';
import type { Readable } from 'node:stream';
import type { Database } from '../app/database/database.types';
import type { EventServices } from '../app/events/events.services';
import type { Config } from '../config/config.types';
import type { CustomPropertiesRepository } from '../custom-properties/custom-properties.repository';
import type { PlanEntitlementsRepository } from '../plan-entitlements/plan-entitlements.repository';
import type { PlanEntitlementDefinitionRegistry } from '../plan-entitlements/plan-entitlements.registry';
import type { PlansRepository } from '../plans/plans.repository';
import type { Logger } from '../shared/logger/logger';
import type { SubscriptionsRepository } from '../subscriptions/subscriptions.repository';
import type { TaggingRulesRepository } from '../tagging-rules/tagging-rules.repository';
import type { TagsRepository } from '../tags/tags.repository';
import type { TaskServices } from '../tasks/tasks.services';
import type { DocumentsRepository } from './documents.repository';
import type { Document } from './documents.types';
import type { StorageService } from '../storage/storage.services';
import type { EncryptionContext } from '../storage/drivers/drivers.models';
import type { CreateDocumentStorageKey } from './document-storage.usecases';
import { PassThrough } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { safely } from '@corentinth/chisels';
import pLimit from 'p-limit';
import { buildCustomPropertiesArray } from '../custom-properties/custom-properties.models';
import { createOrganizationDocumentStorageLimitReachedError } from '../organizations/organizations.errors';
import { createOrganizationsRepository } from '../organizations/organizations.repository';
import { getOrganizationStorageLimits } from '../organizations/organizations.usecases';
import { createPlanEntitlementsRepository } from '../plan-entitlements/plan-entitlements.repository';
import { createPlanEntitlementDefinitionRegistry } from '../plan-entitlements/plan-entitlements.registry';
import { createPlansRepository } from '../plans/plans.repository';
import { createError } from '../shared/errors/errors';
import { createLogger } from '../shared/logger/logger';
import { createByteCounter } from '../shared/streams/byte-counter';
import { createSha256HashTransformer } from '../shared/streams/stream-hash';
import { collectStreamToFile } from '../shared/streams/stream.convertion';
import { isNil } from '../shared/utils';
import { createSubscriptionsRepository } from '../subscriptions/subscriptions.repository';
import { createTaggingRulesRepository } from '../tagging-rules/tagging-rules.repository';
import { applyTaggingRules } from '../tagging-rules/tagging-rules.usecases';
import { createTagsRepository } from '../tags/tags.repository';
import {
createDocumentAlreadyExistsError,
createDocumentNotDeletedError,
createDocumentNotFoundError,
createDocumentSizeTooLargeError,
} from './documents.errors';
import {
formatDocumentForApi,
generateDocumentId as generateDocumentIdImpl,
} from './documents.models';
import { createDocumentsRepository } from './documents.repository';
import { buildCreateDocumentStorageKey } from './document-storage.usecases';
import { buildResolveStoragePatternContext } from './storage-patterns/storage-pattern.usecases';
import type { ExtractDocumentTextUsecase } from './content-extraction/content-extraction.usecases';
type DocumentStorageContext = {
storageKey: string;
} & EncryptionContext;
export async function createDocument({
fileStream,
fileName,
mimeType,
userId,
organizationId,
ocrLanguages = [],
isContentExtractionEnabled = true,
createDocumentStorageKey,
documentsRepository,
documentsStorageService,
generateDocumentId = generateDocumentIdImpl,
plansRepository,
subscriptionsRepository,
planEntitlementsRepository,
planEntitlementDefinitionRegistry,
taggingRulesRepository,
tagsRepository,
eventServices,
taskServices,
logger = createLogger({ namespace: 'documents:usecases' }),
}: {
fileStream: Readable;
fileName: string;
mimeType: string;
userId?: string;
organizationId: string;
ocrLanguages?: string[];
isContentExtractionEnabled?: boolean;
createDocumentStorageKey: CreateDocumentStorageKey;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
generateDocumentId?: () => string;
plansRepository: PlansRepository;
subscriptionsRepository: SubscriptionsRepository;
planEntitlementsRepository: PlanEntitlementsRepository;
planEntitlementDefinitionRegistry: PlanEntitlementDefinitionRegistry;
taggingRulesRepository: TaggingRulesRepository;
tagsRepository: TagsRepository;
eventServices: EventServices;
taskServices: TaskServices;
logger?: Logger;
}) {
const { availableDocumentStorageBytes, maxFileSize } = await getOrganizationStorageLimits({
organizationId,
plansRepository,
subscriptionsRepository,
planEntitlementsRepository,
planEntitlementDefinitionRegistry,
documentsRepository,
});
const documentId = generateDocumentId();
const createdAt = new Date();
const { storageKey } = await createDocumentStorageKey({
documentId,
documentName: fileName,
documentDate: null,
documentCreatedAt: createdAt,
organizationId,
});
const { tap: hashStream, getHash } = createSha256HashTransformer();
// Stream that will count the bytes of the file and throw an error if the file size exceeds the organization storage limit
const { tap: byteCountStream, getByteCount } = createByteCounter({
onByteCountChange: async ({ byteCount, destroy }) => {
if (byteCount > availableDocumentStorageBytes) {
destroy({ error: createOrganizationDocumentStorageLimitReachedError() });
}
if (byteCount > maxFileSize) {
destroy({ error: createDocumentSizeTooLargeError() });
}
},
});
// Create a PassThrough stream that will be used for saving the file
// This allows us to use pipeline for better error handling
const outputStream = new PassThrough();
const streamProcessingPromise = pipeline(fileStream, hashStream, byteCountStream, outputStream);
// We optimistically save the file to leverage streaming, if the file already exists, we will delete it
const [encryptionMetadata] = await Promise.all([
documentsStorageService.saveFile({
fileStream: outputStream,
storageKey,
mimeType,
fileName,
}),
streamProcessingPromise,
]);
const hash = getHash();
const size = getByteCount();
// Early check to avoid saving the file and then realizing it already exists with the db constraint
const { document: existingDocument } =
await documentsRepository.getOrganizationDocumentBySha256Hash({
sha256Hash: hash,
organizationId,
});
const { document } = existingDocument
? await handleExistingDocument({
existingDocument,
fileName,
organizationId,
documentsRepository,
newDocumentStorageKey: storageKey,
tagsRepository,
taggingRulesRepository,
eventServices,
documentsStorageService,
logger,
})
: await createNewDocument({
createdAt,
newFileStorageContext: { storageKey, ...encryptionMetadata },
fileName,
size,
mimeType,
hash,
userId,
organizationId,
documentsRepository,
documentsStorageService,
plansRepository,
subscriptionsRepository,
planEntitlementsRepository,
planEntitlementDefinitionRegistry,
documentId,
taskServices,
ocrLanguages,
isContentExtractionEnabled,
logger,
});
eventServices.emitEvent({
eventName: 'document.created',
payload: { document },
});
return { document };
}
export type CreateDocumentUsecase = Awaited<ReturnType<typeof createDocumentCreationUsecase>>;
export type DocumentUsecaseDependencies = Omit<
Parameters<typeof createDocument>[0],
'fileStream' | 'fileName' | 'mimeType' | 'userId' | 'organizationId'
>;
export function createDocumentCreationUsecase({
db,
config,
taskServices,
documentsStorageService,
eventServices,
...initialDeps
}: {
db: Database;
taskServices: TaskServices;
documentsStorageService: StorageService;
eventServices: EventServices;
config: Config;
} & Partial<DocumentUsecaseDependencies>) {
const deps = {
documentsRepository: initialDeps.documentsRepository ?? createDocumentsRepository({ db }),
plansRepository: initialDeps.plansRepository ?? createPlansRepository({ config }),
subscriptionsRepository:
initialDeps.subscriptionsRepository ?? createSubscriptionsRepository({ db }),
planEntitlementsRepository:
initialDeps.planEntitlementsRepository ?? createPlanEntitlementsRepository({ db }),
planEntitlementDefinitionRegistry:
initialDeps.planEntitlementDefinitionRegistry ??
createPlanEntitlementDefinitionRegistry({ config }),
taggingRulesRepository:
initialDeps.taggingRulesRepository ?? createTaggingRulesRepository({ db }),
tagsRepository: initialDeps.tagsRepository ?? createTagsRepository({ db }),
createDocumentStorageKey:
initialDeps.createDocumentStorageKey ??
buildCreateDocumentStorageKey({
storagePatternConfig: config.documentsStorage.pattern,
documentsStorageService,
resolveStoragePatternContext: buildResolveStoragePatternContext({
organizationsRepository: createOrganizationsRepository({ db }),
}),
}),
ocrLanguages: initialDeps.ocrLanguages ?? config.documents.ocrLanguages,
isContentExtractionEnabled:
initialDeps.isContentExtractionEnabled ?? config.documents.isContentExtractionEnabled,
generateDocumentId: initialDeps.generateDocumentId,
logger: initialDeps.logger,
};
return async (args: {
fileStream: Readable;
fileName: string;
mimeType: string;
userId?: string;
organizationId: string;
}) => createDocument({ taskServices, documentsStorageService, eventServices, ...args, ...deps });
}
async function handleExistingDocument({
existingDocument,
fileName,
userId,
organizationId,
documentsRepository,
tagsRepository,
taggingRulesRepository,
eventServices,
documentsStorageService,
newDocumentStorageKey,
logger,
}: {
existingDocument: Document;
fileName: string;
userId?: string;
organizationId: string;
documentsRepository: DocumentsRepository;
tagsRepository: TagsRepository;
taggingRulesRepository: TaggingRulesRepository;
eventServices: EventServices;
documentsStorageService: StorageService;
newDocumentStorageKey: string;
logger: Logger;
}) {
// Delete the newly uploaded file since we'll be using the existing document's file
await documentsStorageService.deleteFile({ storageKey: newDocumentStorageKey });
if (!existingDocument.isDeleted) {
throw createDocumentAlreadyExistsError();
}
logger.info(
{ documentId: existingDocument.id },
'Document already exists, restoring for deduplication',
);
const [, { document: restoredDocument }] = await Promise.all([
tagsRepository.removeAllTagsFromDocument({ documentId: existingDocument.id }),
documentsRepository.restoreDocument({
documentId: existingDocument.id,
organizationId,
name: fileName,
userId,
}),
]);
await applyTaggingRules({
document: restoredDocument,
taggingRulesRepository,
tagsRepository,
eventServices,
});
return { document: restoredDocument };
}
async function createNewDocument({
createdAt,
fileName,
size,
mimeType,
hash,
userId,
organizationId,
plansRepository,
subscriptionsRepository,
planEntitlementsRepository,
planEntitlementDefinitionRegistry,
documentsRepository,
documentsStorageService,
newFileStorageContext,
documentId,
taskServices,
ocrLanguages = [],
isContentExtractionEnabled = true,
logger,
}: {
createdAt: Date;
fileName: string;
size: number;
mimeType: string;
hash: string;
userId?: string;
organizationId: string;
documentId: string;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
plansRepository: PlansRepository;
subscriptionsRepository: SubscriptionsRepository;
planEntitlementsRepository: PlanEntitlementsRepository;
planEntitlementDefinitionRegistry: PlanEntitlementDefinitionRegistry;
newFileStorageContext: DocumentStorageContext;
taskServices: TaskServices;
ocrLanguages?: string[];
isContentExtractionEnabled?: boolean;
logger: Logger;
}) {
// TODO: wrap in a transaction
// Recheck for quota after saving the file to the storage
const { availableDocumentStorageBytes } = await getOrganizationStorageLimits({
organizationId,
plansRepository,
subscriptionsRepository,
planEntitlementsRepository,
planEntitlementDefinitionRegistry,
documentsRepository,
});
if (size > availableDocumentStorageBytes) {
logger.error(
{ size, availableDocumentStorageBytes },
'Document size exceeds organization storage limit after being saved',
);
await documentsStorageService.deleteFile({ storageKey: newFileStorageContext.storageKey });
throw createOrganizationDocumentStorageLimitReachedError();
}
const [result, error] = await safely(
documentsRepository.saveOrganizationDocument({
id: documentId,
createdAt,
name: fileName,
organizationId,
originalName: fileName,
createdBy: userId,
originalSize: size,
originalStorageKey: newFileStorageContext.storageKey,
fileEncryptionAlgorithm: newFileStorageContext.fileEncryptionAlgorithm,
fileEncryptionKekVersion: newFileStorageContext.fileEncryptionKekVersion,
fileEncryptionKeyWrapped: newFileStorageContext.fileEncryptionKeyWrapped,
mimeType,
originalSha256Hash: hash,
}),
);
if (error) {
logger.error({ error }, 'Error while creating document');
// If the document is not saved, delete the file from the storage
await documentsStorageService.deleteFile({ storageKey: newFileStorageContext.storageKey });
logger.error({ error }, 'Stored document file deleted because of error');
throw error;
}
const { document } = result;
if (isContentExtractionEnabled) {
await taskServices.scheduleJob({
taskName: 'extract-document-file-content',
data: { documentId, organizationId, ocrLanguages },
});
}
logger.info({ documentId, userId, organizationId, mimeType }, 'Document created');
return { document };
}
export async function getDocumentOrThrow({
documentId,
organizationId,
documentsRepository,
}: {
documentId: string;
organizationId: string;
documentsRepository: DocumentsRepository;
}) {
const { document } = await documentsRepository.getDocumentById({ documentId, organizationId });
if (!document) {
throw createDocumentNotFoundError();
}
return { document };
}
export async function ensureDocumentExists({
documentId,
organizationId,
documentsRepository,
}: {
documentId: string;
organizationId: string;
documentsRepository: DocumentsRepository;
}) {
await getDocumentOrThrow({ documentId, organizationId, documentsRepository });
}
export async function hardDeleteDocument({
document,
documentsRepository,
documentsStorageService,
eventServices,
}: {
document: Pick<Document, 'id' | 'originalStorageKey' | 'organizationId'>;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
eventServices: EventServices;
}) {
await Promise.all([
documentsRepository.hardDeleteDocument({ documentId: document.id }),
documentsStorageService.deleteFile({ storageKey: document.originalStorageKey }),
]);
eventServices.emitEvent({
eventName: 'document.deleted',
payload: { documentId: document.id, organizationId: document.organizationId },
});
}
export async function deleteExpiredDocuments({
documentsRepository,
documentsStorageService,
eventServices,
config,
now = new Date(),
logger = createLogger({ namespace: 'documents:deleteExpiredDocuments' }),
}: {
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
eventServices: EventServices;
config: Config;
now?: Date;
logger?: Logger;
}) {
const { documents } = await documentsRepository.getExpiredDeletedDocuments({
expirationDelayInDays: config.documents.deletedDocumentsRetentionDays,
now,
});
const limit = pLimit(10);
await Promise.all(
documents.map(async (document) =>
limit(async () => {
const [, error] = await safely(
hardDeleteDocument({
document,
documentsRepository,
documentsStorageService,
eventServices,
}),
);
if (error) {
logger.error({ document, error }, 'Error while deleting expired document');
}
}),
),
);
return {
deletedDocumentsCount: documents.length,
};
}
export async function deleteTrashDocument({
documentId,
organizationId,
documentsRepository,
documentsStorageService,
eventServices,
}: {
documentId: string;
organizationId: string;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
eventServices: EventServices;
}) {
const { document } = await documentsRepository.getDocumentById({ documentId, organizationId });
if (!document) {
throw createDocumentNotFoundError();
}
if (!document.isDeleted) {
throw createDocumentNotDeletedError();
}
await hardDeleteDocument({
document,
documentsRepository,
documentsStorageService,
eventServices,
});
}
export async function deleteAllTrashDocuments({
organizationId,
documentsRepository,
documentsStorageService,
eventServices,
}: {
organizationId: string;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
eventServices: EventServices;
}) {
const { documents } = await documentsRepository.getAllOrganizationTrashDocuments({
organizationId,
});
// TODO: refactor to use batching and transaction
const limit = pLimit(10);
await Promise.all(
documents.map(async (document) =>
limit(async () => {
await hardDeleteDocument({
document,
documentsRepository,
documentsStorageService,
eventServices,
});
}),
),
);
}
export async function extractAndSaveDocumentFileContent({
documentId,
organizationId,
documentsRepository,
documentsStorageService,
taggingRulesRepository,
tagsRepository,
eventServices,
extractDocumentText,
}: {
documentId: string;
organizationId: string;
documentsRepository: DocumentsRepository;
documentsStorageService: StorageService;
taggingRulesRepository: TaggingRulesRepository;
tagsRepository: TagsRepository;
eventServices: EventServices;
extractDocumentText: ExtractDocumentTextUsecase;
}) {
const { document } = await documentsRepository.getDocumentById({ documentId, organizationId });
if (!document) {
throw createDocumentNotFoundError();
}
const { fileStream } = await documentsStorageService.getFileStream({
storageKey: document.originalStorageKey,
fileEncryptionAlgorithm: document.fileEncryptionAlgorithm,
fileEncryptionKekVersion: document.fileEncryptionKekVersion,
fileEncryptionKeyWrapped: document.fileEncryptionKeyWrapped,
});
const { file } = await collectStreamToFile({
fileStream,
fileName: document.name,
mimeType: document.mimeType,
});
const { text } = await extractDocumentText({ file });
const { document: updatedDocument } = await updateDocument({
documentId,
organizationId,
changes: { content: text },
documentsRepository,
eventServices,
});
if (isNil(updatedDocument)) {
// This should never happen, but for type safety
throw createDocumentNotFoundError();
}
await applyTaggingRules({
document: updatedDocument,
taggingRulesRepository,
tagsRepository,
eventServices,
});
return { document: updatedDocument };
}
export async function trashDocument({
documentId,
organizationId,
userId,
documentsRepository,
eventServices,
}: {
documentId: string;
organizationId: string;
userId: string;
documentsRepository: DocumentsRepository;
eventServices: EventServices;
}) {
await documentsRepository.softDeleteDocument({ documentId, organizationId, userId });
eventServices.emitEvent({
eventName: 'documents.trashed',
payload: { documentIds: [documentId], organizationId, trashedBy: userId },
});
}
export async function restoreDocument({
documentId,
organizationId,
userId,
documentsRepository,
eventServices,
}: {
documentId: string;
organizationId: string;
userId: string;
documentsRepository: DocumentsRepository;
eventServices: EventServices;
}) {
await documentsRepository.restoreDocument({ documentId, organizationId });
eventServices.emitEvent({
eventName: 'document.restored',
payload: { documentId, organizationId, restoredBy: userId },
});
}
export async function enrichAndFormatDocumentsForApi({
documents,
tagsRepository,
customPropertiesRepository,
}: {
documents: Array<PartialBy<Document, 'content'>>;
tagsRepository: TagsRepository;
customPropertiesRepository: CustomPropertiesRepository;
}) {
if (documents.length === 0) {
return { enrichedDocuments: [] };
}
const { organizationId } = documents[0] ?? {};
if (isNil(organizationId)) {
throw createError({
message: 'Organization ID is required to enrich documents',
code: 'organization_id_required',
statusCode: 500,
});
}
const uniqueOrganizationIds = new Set(documents.map((d) => d.organizationId));
if (uniqueOrganizationIds.size > 1) {
throw createError({
message: 'All documents must belong to the same organization to be enriched',
code: 'multiple_organization_ids',
statusCode: 500,
});
}
const documentIds = documents.map((d) => d.id);
const [{ tagsByDocumentId }, { valuesByDocumentId }, { propertyDefinitions }] = await Promise.all(
[
tagsRepository.getTagsByDocumentIds({ documentIds }),
customPropertiesRepository.getCustomPropertyValuesByDocumentIds({ documentIds }),
customPropertiesRepository.getOrganizationPropertyDefinitions({ organizationId }),
],
);
return {
enrichedDocuments: documents.map((document) => ({
...formatDocumentForApi({ document }),
tags: tagsByDocumentId[document.id] ?? [],
customProperties: buildCustomPropertiesArray({
rawValues: valuesByDocumentId[document.id] ?? [],
propertyDefinitions,
}),
})),
};
}
export async function enrichAndFormatDocumentForApi({
document,
tagsRepository,
customPropertiesRepository,
}: {
document: PartialBy<Document, 'content'>;
tagsRepository: TagsRepository;
customPropertiesRepository: CustomPropertiesRepository;
}) {
const { enrichedDocuments } = await enrichAndFormatDocumentsForApi({
documents: [document],
tagsRepository,
customPropertiesRepository,
});
const [enrichedDocument] = enrichedDocuments;
if (!enrichedDocument) {
// This should never happen, but for type safety
throw createError({
message: 'Document enrichment failed',
code: 'document_enrichment_failed',
statusCode: 500,
});
}
return { enrichedDocument };
}
export async function updateDocument({
documentId,
organizationId,
userId,
documentsRepository,
eventServices,
changes,
}: {
documentId: string;
organizationId: string;
userId?: string;
documentsRepository: DocumentsRepository;
eventServices: EventServices;
changes: {
name?: string;
content?: string;
documentDate?: Date | null;
notes?: string;
};
}) {
// It throws if the document does not exist
const { document } = await documentsRepository.updateDocument({
documentId,
organizationId,
...changes,
});
eventServices.emitEvent({
eventName: 'document.updated',
payload: { userId, changes, document },
});
return { document };
}