The minimalistic document archiving platform.
refactor(search): removed database fts triggers (#683)
Corentin Thomassetd6f71ba5ecb8
diff --git a/apps/papra-server/src/migrations/list/0013-drop-fts-5-triggers.migration.ts b/apps/papra-server/src/migrations/list/0013-drop-fts-5-triggers.migration.tsnew file mode 100644index 00000000..2b0431f8--- /dev/null+++ b/apps/papra-server/src/migrations/list/0013-drop-fts-5-triggers.migration.ts@@ -0,0 +1,34 @@+import type { Migration } from '../migrations.types';+import { sql } from 'drizzle-orm';++export const dropFts5TriggersMigration = {+ name: 'drop-fts-5-triggers',++ up: async ({ db }) => {+ await db.batch([+ db.run(sql`DROP TRIGGER IF EXISTS trigger_documents_fts_insert`),+ db.run(sql`DROP TRIGGER IF EXISTS trigger_documents_fts_update`),+ db.run(sql`DROP TRIGGER IF EXISTS trigger_documents_fts_delete`),+ ]);+ },++ down: async ({ db }) => {+ await db.batch([+ db.run(sql`+ CREATE TRIGGER IF NOT EXISTS trigger_documents_fts_insert AFTER INSERT ON documents BEGIN+ INSERT INTO documents_fts(id, name, original_name, content) VALUES (new.id, new.name, new.original_name, new.content);+ END+ `),+ db.run(sql`+ CREATE TRIGGER IF NOT EXISTS trigger_documents_fts_update AFTER UPDATE ON documents BEGIN+ UPDATE documents_fts SET name = new.name, original_name = new.original_name, content = new.content WHERE id = new.id;+ END+ `),+ db.run(sql`+ CREATE TRIGGER IF NOT EXISTS trigger_documents_fts_delete AFTER DELETE ON documents BEGIN+ DELETE FROM documents_fts WHERE id = old.id;+ END+ `),+ ]);+ },+} satisfies Migration;diff --git a/apps/papra-server/src/migrations/migrations.registry.test.ts b/apps/papra-server/src/migrations/migrations.registry.test.tsindex 902cc6e3..ff137ceb 100644--- a/apps/papra-server/src/migrations/migrations.registry.test.ts+++ b/apps/papra-server/src/migrations/migrations.registry.test.ts@@ -121,10 +121,7 @@ describe('migrations registry', () => {CREATE TABLE "users" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "email" text NOT NULL, "email_verified" integer DEFAULT false NOT NULL, "name" text, "image" text, "max_organization_count" integer );CREATE TABLE "webhook_deliveries" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "webhook_id" text NOT NULL, "event_name" text NOT NULL, "request_payload" text NOT NULL, "response_payload" text NOT NULL, "response_status" integer NOT NULL, FOREIGN KEY ("webhook_id") REFERENCES "webhooks"("id") ON UPDATE cascade ON DELETE cascade );CREATE TABLE "webhook_events" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "webhook_id" text NOT NULL, "event_name" text NOT NULL, FOREIGN KEY ("webhook_id") REFERENCES "webhooks"("id") ON UPDATE cascade ON DELETE cascade );- CREATE TABLE "webhooks" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "name" text NOT NULL, "url" text NOT NULL, "secret" text, "enabled" integer DEFAULT true NOT NULL, "created_by" text, "organization_id" text, FOREIGN KEY ("created_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON UPDATE cascade ON DELETE cascade );- CREATE TRIGGER trigger_documents_fts_delete AFTER DELETE ON documents BEGIN DELETE FROM documents_fts WHERE id = old.id; END;- CREATE TRIGGER trigger_documents_fts_insert AFTER INSERT ON documents BEGIN INSERT INTO documents_fts(id, name, original_name, content) VALUES (new.id, new.name, new.original_name, new.content); END;- CREATE TRIGGER trigger_documents_fts_update AFTER UPDATE ON documents BEGIN UPDATE documents_fts SET name = new.name, original_name = new.original_name, content = new.content WHERE id = new.id; END;"+ CREATE TABLE "webhooks" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "name" text NOT NULL, "url" text NOT NULL, "secret" text, "enabled" integer DEFAULT true NOT NULL, "created_by" text, "organization_id" text, FOREIGN KEY ("created_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON UPDATE cascade ON DELETE cascade );"`);});diff --git a/apps/papra-server/src/migrations/migrations.registry.ts b/apps/papra-server/src/migrations/migrations.registry.tsindex da7a862f..ccb2ebbc 100644--- a/apps/papra-server/src/migrations/migrations.registry.ts+++ b/apps/papra-server/src/migrations/migrations.registry.ts@@ -15,6 +15,8 @@ import { documentFileEncryptionMigration } from './list/0010-document-file-encryimport { softDeleteOrganizationsMigration } from './list/0011-soft-delete-organizations.migration';import { taggingRuleConditionMatchModeMigration } from './list/0012-tagging-rule-condition-match-mode.migration';+import { dropFts5TriggersMigration } from './list/0013-drop-fts-5-triggers.migration';+export const migrations: Migration[] = [initialSchemaSetupMigration,documentsFtsMigration,@@ -28,4 +30,5 @@ export const migrations: Migration[] = [documentFileEncryptionMigration,softDeleteOrganizationsMigration,taggingRuleConditionMatchModeMigration,+ dropFts5TriggersMigration,];diff --git a/apps/papra-server/src/modules/app/events/events.handlers.ts b/apps/papra-server/src/modules/app/events/events.handlers.tsindex 2629e35d..eebcac45 100644--- a/apps/papra-server/src/modules/app/events/events.handlers.ts+++ b/apps/papra-server/src/modules/app/events/events.handlers.ts@@ -1,6 +1,8 @@+import type { DocumentSearchServices } from '../../documents/document-search/document-search.types';import type { TrackingServices } from '../../tracking/tracking.services';import type { Database } from '../database/database.types';import type { EventServices } from './events.services';+import { registerSyncDocumentSearchEventHandlers } from '../../documents/document-search/events/sync-document-search.handlers';import { registerInsertActivityLogOnDocumentCreatedHandler } from '../../documents/events/activity-log.document-created';import { registerInsertActivityLogOnDocumentRestoredHandler } from '../../documents/events/activity-log.document-restored';import { registerInsertActivityLogOnDocumentTrashedHandler } from '../../documents/events/activity-log.document-trashed';@@ -11,7 +13,7 @@ import { registerTriggerWebhooksOnDocumentTrashedHandler } from '../../documentsimport { registerTriggerWebhooksOnDocumentUpdatedHandler } from '../../documents/events/webhook.document-updated';import { registerTrackingUserCreatedEventHandler } from '../../users/event-handlers/tracking.user-created';-export function registerEventHandlers(deps: { trackingServices: TrackingServices; eventServices: EventServices; db: Database }) {+export function registerEventHandlers(deps: { trackingServices: TrackingServices; eventServices: EventServices; db: Database; documentSearchServices: DocumentSearchServices }) {registerTrackingUserCreatedEventHandler(deps);registerTriggerWebhooksOnDocumentCreatedHandler(deps);registerInsertActivityLogOnDocumentCreatedHandler(deps);@@ -21,4 +23,5 @@ export function registerEventHandlers(deps: { trackingServices: TrackingServicesregisterInsertActivityLogOnDocumentRestoredHandler(deps);registerTriggerWebhooksOnDocumentUpdatedHandler(deps);registerInsertActivityLogOnDocumentUpdatedHandler(deps);+ registerSyncDocumentSearchEventHandlers(deps);}diff --git a/apps/papra-server/src/modules/app/server.test-utils.ts b/apps/papra-server/src/modules/app/server.test-utils.tsindex da6ad516..ee7ed2d0 100644--- a/apps/papra-server/src/modules/app/server.test-utils.ts+++ b/apps/papra-server/src/modules/app/server.test-utils.ts@@ -29,7 +29,7 @@ export function createTestServerDependencies(overrides: Partial<GlobalDependenciconst subscriptionsServices = overrides.subscriptionsServices ?? createSubscriptionsServices({ config });const documentSearchServices = overrides.documentSearchServices ?? createDocumentSearchServices({ db, config });- registerEventHandlers({ eventServices, trackingServices, db });+ registerEventHandlers({ eventServices, trackingServices, db, documentSearchServices });return {config,diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.tsindex ac267c7e..7e4d8c4a 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts@@ -8,6 +8,7 @@ export function createDatabaseFts5DocumentSearchServices({ db }: { db: Databasereturn {name: DATABASE_FTS5_DOCUMENT_SEARCH_PROVIDER_NAME,+searchDocuments: async ({ searchQuery, organizationId, pageIndex, pageSize }) => {const { documents } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });@@ -17,5 +18,18 @@ export function createDatabaseFts5DocumentSearchServices({ db }: { db: Database},};},++ indexDocument: async ({ document }) => {+ await documentsSearchRepository.indexDocument({ document });+ },++ updateDocument: async ({ document, documentId }) => {+ await documentsSearchRepository.updateDocument({ documentId, document });+ },++ deleteDocument: async ({ documentId }) => {+ await documentsSearchRepository.deleteDocument({ documentId });+ },+};}diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.tsindex 22c7e7ca..8b146988 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts@@ -1,36 +1,219 @@import { describe, expect, test } from 'vitest';import { createInMemoryDatabase } from '../../../app/database/database.test-utils';-import { ORGANIZATION_ROLES } from '../../../organizations/organizations.constants';+import { documentsTable } from '../../documents.table';import { createDocumentSearchRepository } from './database-fts5.repository';describe('database-fts5 repository', () => {describe('searchOrganizationDocuments', () => {test('provides full text search on document name, original name, and content', async () => {+ const documents = [+ { id: 'doc-1', organizationId: 'organization-1', name: 'Document 1', originalName: 'document-1.pdf', content: 'lorem ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1', isDeleted: false },+ { id: 'doc-2', organizationId: 'organization-1', name: 'File 2', originalName: 'document-2.pdf', content: 'lorem', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash2', isDeleted: false },+ { id: 'doc-3', organizationId: 'organization-1', name: 'File 3', originalName: 'document-3.pdf', content: 'ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash3', isDeleted: false },+ ];+const { db } = await createInMemoryDatabase({- users: [{ id: 'user-1', email: '[email protected]' }],organizations: [{ id: 'organization-1', name: 'Organization 1' }],- organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],- documents: [- { id: 'doc-1', organizationId: 'organization-1', createdBy: 'user-1', name: 'Document 1', originalName: 'document-1.pdf', content: 'lorem ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1' },- { id: 'doc-2', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 2', originalName: 'document-2.pdf', content: 'lorem', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash2' },- { id: 'doc-3', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 3', originalName: 'document-3.pdf', content: 'ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash3' },- ],+ documents,+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ await Promise.all(+ documents.map(async document => documentsSearchRepository.indexDocument({ document })),+ );++ const { documents: searchResults } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'lorem',+ pageIndex: 0,+ pageSize: 10,});- // Rebuild the FTS index since we are using an in-memory database- await db.$client.execute(`INSERT INTO documents_fts(documents_fts) VALUES('rebuild');`);+ expect(searchResults).to.have.length(2);+ expect(searchResults.map(doc => doc.id)).to.eql(['doc-2', 'doc-1']);+ });+ });++ describe('indexDocument', () => {+ test('adds a document to the FTS index and makes it searchable', async () => {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ });const documentsSearchRepository = createDocumentSearchRepository({ db });+ // Insert a document in the documents table (no automatic FTS indexing after migration)+ await db.insert(documentsTable).values({+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'New Document',+ originalName: 'new-doc.pdf',+ content: 'searchable content here',+ originalStorageKey: 'storage-key',+ mimeType: 'application/pdf',+ originalSha256Hash: 'hash1',+ }).execute();++ // Manually index the document to FTS+ await documentsSearchRepository.indexDocument({+ document: {+ id: 'doc-1',+ name: 'New Document',+ isDeleted: false,+ organizationId: 'organization-1',+ originalName: 'new-doc.pdf',+ content: 'searchable content here',+ },+ });++ // Verify document is searchableconst { documents } = await documentsSearchRepository.searchOrganizationDocuments({organizationId: 'organization-1',- searchQuery: 'lorem',+ searchQuery: 'searchable',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(documents).to.have.length(1);+ expect(documents[0]?.id).to.equal('doc-1');+ });+ });++ describe('updateDocument', () => {+ test('updates document fields in the FTS index to reflect changes in search results', async () => {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ // Insert document and manually index it with original content+ await db.insert(documentsTable).values({+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'Original Name',+ originalName: 'original.pdf',+ content: 'original content',+ originalStorageKey: '',+ mimeType: 'application/pdf',+ originalSha256Hash: 'hash1',+ }).execute();++ await documentsSearchRepository.indexDocument({+ document: {+ id: 'doc-1',+ name: 'Original Name',+ originalName: 'original.pdf',+ content: 'original content',+ isDeleted: false,+ organizationId: 'organization-1',+ },+ });++ // Update the FTS index with new content+ await documentsSearchRepository.updateDocument({+ documentId: 'doc-1',+ document: {+ name: 'Updated Name',+ originalName: 'updated.pdf',+ content: 'updated content with new keywords',+ },+ });++ const resultsWithOldKeyword = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'original',+ pageIndex: 0,+ pageSize: 10,+ });++ const resultsWithNewKeyword = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'updated',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(resultsWithOldKeyword.documents).to.have.length(0);+ expect(resultsWithNewKeyword.documents).to.have.length(1);+ expect(resultsWithNewKeyword.documents[0]?.id).to.equal('doc-1');+ });+ });++ describe('deleteDocument', () => {+ test('removes a document from the FTS index and makes it unsearchable', async () => {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ // Insert documents and manually index them+ await db.insert(documentsTable).values([+ {+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'Document to Delete',+ originalName: 'delete-me.pdf',+ content: 'this will be deleted',+ originalStorageKey: '',+ mimeType: 'application/pdf',+ originalSha256Hash: 'hash1',+ },+ {+ id: 'doc-2',+ organizationId: 'organization-1',+ name: 'Document to Keep',+ originalName: 'keep-me.pdf',+ content: 'this will stay',+ originalStorageKey: '',+ mimeType: 'application/pdf',+ originalSha256Hash: 'hash2',+ },+ ]).execute();++ await documentsSearchRepository.indexDocument({+ document: {+ id: 'doc-1',+ name: 'Document to Delete',+ originalName: 'delete-me.pdf',+ content: 'this will be deleted',+ isDeleted: false,+ organizationId: 'organization-1',+ },+ });++ await documentsSearchRepository.indexDocument({+ document: {+ id: 'doc-2',+ name: 'Document to Keep',+ originalName: 'keep-me.pdf',+ content: 'this will stay',+ isDeleted: false,+ organizationId: 'organization-1',+ },+ });++ const beforeDelete = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'deleted',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(beforeDelete.documents).to.have.length(1);++ await documentsSearchRepository.deleteDocument({ documentId: 'doc-1' });++ const afterDelete = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'deleted',pageIndex: 0,pageSize: 10,});- expect(documents).to.have.length(2);- expect(documents.map(doc => doc.id)).to.eql(['doc-2', 'doc-1']);+ expect(afterDelete.documents).to.have.length(0);});});});diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.tsindex 2cecfd14..e4a47812 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts@@ -1,13 +1,19 @@import type { Database } from '../../../app/database/database.types';+import type { DocumentSearchableData } from '../document-search.types';import { injectArguments } from '@corentinth/chisels';-import { sql } from 'drizzle-orm';+import { and, eq, getTableColumns, sql } from 'drizzle-orm';+import { omitUndefined } from '../../../shared/utils';import { documentsTable } from '../../documents.table';+import { documentsFtsTable } from './database-fts5.tables';export type DocumentSearchRepository = ReturnType<typeof createDocumentSearchRepository>;export function createDocumentSearchRepository({ db }: { db: Database }) {return injectArguments({searchOrganizationDocuments,+ indexDocument,+ updateDocument,+ deleteDocument,}, { db });}@@ -17,18 +23,54 @@ async function searchOrganizationDocuments({ organizationId, searchQuery, pageInconst cleanedSearchQuery = searchQuery.replace(/"/g, '').replace(/\*/g, '').trim();const formattedSearchQuery = cleanedSearchQuery.includes(' ') ? cleanedSearchQuery : `${cleanedSearchQuery}*`;- const result = await db.run(sql`- SELECT * FROM ${documentsTable}- JOIN documents_fts ON documents_fts.id = ${documentsTable.id}- WHERE ${documentsTable.organizationId} = ${organizationId}- AND ${documentsTable.isDeleted} = 0- AND documents_fts MATCH ${formattedSearchQuery}- ORDER BY rank- LIMIT ${pageSize}- OFFSET ${pageIndex * pageSize}- `);-- return {- documents: result.rows as unknown as (typeof documentsTable.$inferSelect)[],- };+ const documents = await db.select(getTableColumns(documentsTable))+ .from(documentsTable)+ .innerJoin(+ documentsFtsTable,+ eq(documentsFtsTable.id, documentsTable.id),+ )+ .where(and(+ eq(documentsTable.organizationId, organizationId),+ eq(documentsTable.isDeleted, false),+ eq(documentsFtsTable, formattedSearchQuery), // Match and eq works the same for FTS5 virtual tables+ ))+ .orderBy(sql`rank`)+ .limit(pageSize)+ .offset(pageIndex * pageSize);++ return { documents };+}++async function indexDocument({ document, db }: { document: DocumentSearchableData; db: Database }) {+ await db+ .insert(documentsFtsTable)+ .values({+ id: document.id,+ name: document.name,+ originalName: document.originalName,+ content: document.content,+ });+}++async function updateDocument({ documentId, document, db }: { documentId: string; document: { content?: string; name?: string; originalName?: string }; db: Database }) {+ const dataToUpdate = omitUndefined({+ name: document.name,+ originalName: document.originalName,+ content: document.content,+ });++ if (Object.keys(dataToUpdate).length === 0) {+ return;+ }++ await db+ .update(documentsFtsTable)+ .set(dataToUpdate)+ .where(eq(documentsFtsTable.id, documentId));+}++async function deleteDocument({ documentId, db }: { documentId: string; db: Database }) {+ await db+ .delete(documentsFtsTable)+ .where(eq(documentsFtsTable.id, documentId));}diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.tsnew file mode 100644index 00000000..c12dcf0f--- /dev/null+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts@@ -0,0 +1,8 @@+import { sqliteTable, text } from 'drizzle-orm/sqlite-core';++export const documentsFtsTable = sqliteTable('documents_fts', {+ id: text('id').notNull(),+ name: text('name').notNull(),+ originalName: text('original_name').notNull(),+ content: text('content').notNull(),+});diff --git a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts b/apps/papra-server/src/modules/documents/document-search/document-search.types.tsindex e51c408f..d007ade9 100644--- a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts+++ b/apps/papra-server/src/modules/documents/document-search/document-search.types.ts@@ -1,3 +1,12 @@+export type DocumentSearchableData = {+ id: string;+ name: string;+ originalName: string;+ content: string;+ isDeleted: boolean;+ organizationId: string;+};+export type DocumentSearchServices = {name: string;searchDocuments: (args: {@@ -6,4 +15,8 @@ export type DocumentSearchServices = {pageIndex: number;pageSize: number;}) => Promise<{ searchResults: { documents: { id: string; name: string }[] } }>;++ indexDocument: (args: { document: DocumentSearchableData }) => Promise<void>;+ updateDocument: (args: { documentId: string; document: Partial<Omit<DocumentSearchableData, 'id'>> }) => Promise<void>;+ deleteDocument: (args: { documentId: string }) => Promise<void>;};diff --git a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.tsnew file mode 100644index 00000000..6468fa3d--- /dev/null+++ b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts@@ -0,0 +1,311 @@+import type { Document } from '../../documents.types';+import type { DocumentSearchServices } from '../document-search.types';+import { createNoopLogger } from '@crowlog/logger';+import { describe, expect, test } from 'vitest';+import { createEventServices } from '../../../app/events/events.services';+import { nextTick } from '../../../shared/async/defer.test-utils';+import { registerSyncDocumentSearchEventHandlers } from './sync-document-search.handlers';++function createTestSearchServices() {+ const methodsArgs = {+ searchDocuments: [] as Parameters<DocumentSearchServices['searchDocuments']>[0][],+ indexDocument: [] as Parameters<DocumentSearchServices['indexDocument']>[0][],+ updateDocument: [] as Parameters<DocumentSearchServices['updateDocument']>[0][],+ deleteDocument: [] as Parameters<DocumentSearchServices['deleteDocument']>[0][],+ };++ const searchServices: DocumentSearchServices = {+ name: 'test-search-service',+ searchDocuments: async (args) => {+ methodsArgs.searchDocuments.push(args);+ return { searchResults: { documents: [] } };+ },+ indexDocument: async (args) => {+ methodsArgs.indexDocument.push(args);+ },+ updateDocument: async (args) => {+ methodsArgs.updateDocument.push(args);+ },+ deleteDocument: async (args) => {+ methodsArgs.deleteDocument.push(args);+ },+ };++ return {+ ...searchServices,+ getMethodsArguments: () => methodsArgs,+ };+}++describe('sync-document-search event handlers', () => {+ describe('registerSyncDocumentSearchEventHandlers', () => {+ test('when document.created event fires, the document is indexed in the search service', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ const document: Document = {+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'Test Document',+ originalName: 'test-document.pdf',+ content: 'searchable content',+ mimeType: 'application/pdf',+ originalStorageKey: 'storage-key',+ originalSha256Hash: 'hash1',+ isDeleted: false,+ createdAt: new Date(),+ updatedAt: new Date(),+ createdBy: 'user-1',+ originalSize: 1024,+ deletedAt: null,+ deletedBy: null,+ fileEncryptionAlgorithm: null,+ fileEncryptionKekVersion: null,+ fileEncryptionKeyWrapped: null,+ };++ eventServices.emitEvent({+ eventName: 'document.created',+ payload: { document },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [{ document }],+ updateDocument: [],+ deleteDocument: [],+ });+ });++ test('when document.updated event fires, the document is updated in the search service with the changes', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ const document: Document = {+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'Updated Document',+ originalName: 'updated-document.pdf',+ content: 'updated content',+ mimeType: 'application/pdf',+ originalStorageKey: 'storage-key',+ originalSha256Hash: 'hash1',+ isDeleted: false,+ createdAt: new Date(),+ updatedAt: new Date(),+ createdBy: 'user-1',+ originalSize: 1024,+ deletedAt: null,+ deletedBy: null,+ fileEncryptionAlgorithm: null,+ fileEncryptionKekVersion: null,+ fileEncryptionKeyWrapped: null,+ };++ const changes = {+ name: 'Updated Document',+ content: 'updated content',+ };++ eventServices.emitEvent({+ eventName: 'document.updated',+ payload: { document, changes, userId: 'user-1' },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [],+ updateDocument: [{+ documentId: 'doc-1',+ document: changes,+ }],+ deleteDocument: [],+ });+ });++ test('when document.trashed event fires, the document is marked as deleted in the search service', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ eventServices.emitEvent({+ eventName: 'document.trashed',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ trashedBy: 'user-1',+ },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [],+ updateDocument: [{+ documentId: 'doc-1',+ document: { isDeleted: true },+ }],+ deleteDocument: [],+ });+ });++ test('when document.restored event fires, the document is marked as not deleted in the search service', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ eventServices.emitEvent({+ eventName: 'document.restored',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ restoredBy: 'user-1',+ },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [],+ updateDocument: [{+ documentId: 'doc-1',+ document: { isDeleted: false },+ }],+ deleteDocument: [],+ });+ });++ test('when document.deleted event fires, the document is removed from the search service', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ eventServices.emitEvent({+ eventName: 'document.deleted',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [],+ updateDocument: [],+ deleteDocument: [{ documentId: 'doc-1' }],+ });+ });++ test('multiple events are handled independently and in sequence', async () => {+ const eventServices = createEventServices({ logger: createNoopLogger() });++ const documentSearchServices = createTestSearchServices();++ registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices });++ const document: Document = {+ id: 'doc-1',+ organizationId: 'organization-1',+ name: 'Test Document',+ originalName: 'test-document.pdf',+ content: 'searchable content',+ mimeType: 'application/pdf',+ originalStorageKey: 'storage-key',+ originalSha256Hash: 'hash1',+ isDeleted: false,+ createdAt: new Date(),+ updatedAt: new Date(),+ createdBy: 'user-1',+ originalSize: 1024,+ deletedAt: null,+ deletedBy: null,+ fileEncryptionAlgorithm: null,+ fileEncryptionKekVersion: null,+ fileEncryptionKeyWrapped: null,+ };++ // Emit multiple events in sequence+ eventServices.emitEvent({+ eventName: 'document.created',+ payload: { document },+ });++ eventServices.emitEvent({+ eventName: 'document.updated',+ payload: {+ document,+ changes: { name: 'Updated Name' },+ userId: 'user-1',+ },+ });++ eventServices.emitEvent({+ eventName: 'document.trashed',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ trashedBy: 'user-1',+ },+ });++ eventServices.emitEvent({+ eventName: 'document.restored',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ restoredBy: 'user-1',+ },+ });++ eventServices.emitEvent({+ eventName: 'document.deleted',+ payload: {+ documentId: 'doc-1',+ organizationId: 'organization-1',+ },+ });++ await nextTick();++ expect(documentSearchServices.getMethodsArguments()).to.eql({+ searchDocuments: [],+ indexDocument: [{ document }],+ updateDocument: [+ {+ documentId: 'doc-1',+ document: { name: 'Updated Name' },+ },+ {+ documentId: 'doc-1',+ document: { isDeleted: true },+ },+ {+ documentId: 'doc-1',+ document: { isDeleted: false },+ },+ ],+ deleteDocument: [{ documentId: 'doc-1' }],+ });+ });+ });+});diff --git a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.ts b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.tsnew file mode 100644index 00000000..4c1c2e64--- /dev/null+++ b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.ts@@ -0,0 +1,50 @@+import type { EventServices } from '../../../app/events/events.services';+import type { DocumentSearchServices } from '../document-search.types';++/**+ * Wires up document events to the search service to asynchronously synchronize the service with document changes.+ */+export function registerSyncDocumentSearchEventHandlers({ eventServices, documentSearchServices }: {+ eventServices: EventServices;+ documentSearchServices: DocumentSearchServices;+}) {+ eventServices.onEvent({+ eventName: 'document.created',+ handlerName: 'index-document-in-search-service',+ async handler({ document }) {+ await documentSearchServices.indexDocument({ document });+ },+ });++ eventServices.onEvent({+ eventName: 'document.updated',+ handlerName: 'update-document-in-search-service',+ async handler({ document, changes }) {+ await documentSearchServices.updateDocument({ document: changes, documentId: document.id });+ },+ });++ eventServices.onEvent({+ eventName: 'document.trashed',+ handlerName: 'mark-document-deleted-in-search-service',+ async handler({ documentId }) {+ await documentSearchServices.updateDocument({ documentId, document: { isDeleted: true } });+ },+ });++ eventServices.onEvent({+ eventName: 'document.restored',+ handlerName: 'restore-document-in-search-service',+ async handler({ documentId }) {+ await documentSearchServices.updateDocument({ documentId, document: { isDeleted: false } });+ },+ });++ eventServices.onEvent({+ eventName: 'document.deleted',+ handlerName: 'remove-document-from-search-service',+ async handler({ documentId }) {+ await documentSearchServices.deleteDocument({ documentId });+ },+ });+}diff --git a/apps/papra-server/src/modules/documents/documents.events.types.ts b/apps/papra-server/src/modules/documents/documents.events.types.tsindex e3e3660c..07c28762 100644--- a/apps/papra-server/src/modules/documents/documents.events.types.ts+++ b/apps/papra-server/src/modules/documents/documents.events.types.ts@@ -5,9 +5,8 @@ export type DocumentEvents = {'document.trashed': { documentId: string; organizationId: string; trashedBy: string }; // Soft deleted by moving to trash'document.restored': { documentId: string; organizationId: string; restoredBy: string };'document.updated': {- documentId: string;- organizationId: string;userId?: string;+ document: Document;changes: {name?: string;content?: string;diff --git a/apps/papra-server/src/modules/documents/documents.table.test.ts b/apps/papra-server/src/modules/documents/documents.table.test.tsdeleted file mode 100644index 4ded2cab..00000000--- a/apps/papra-server/src/modules/documents/documents.table.test.ts+++ /dev/null@@ -1,185 +0,0 @@-import { eq, sql } from 'drizzle-orm';-import { describe, expect, test } from 'vitest';-import { createInMemoryDatabase } from '../app/database/database.test-utils';-import { ORGANIZATION_ROLES } from '../organizations/organizations.constants';-import { documentsTable } from './documents.table';--describe('documents table', () => {- describe('table documents_fts', () => {- describe('the documents_fts table is synchronized with the documents table using triggers', async () => {- test('when inserting a document, a corresponding row is inserted in the documents_fts table', async () => {- const { db } = await createInMemoryDatabase({- users: [{ id: 'user-1', email: '[email protected]' }],- organizations: [{ id: 'organization-1', name: 'Organization 1' }],- organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],- });-- await db.insert(documentsTable).values([- {- id: 'document-1',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Document 1',- originalName: 'document-1.pdf',- originalStorageKey: 'document-1.pdf',- content: 'lorem ipsum',- originalSha256Hash: 'hash1',- },- {- id: 'document-2',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Photo 1',- originalName: 'photo-1.jpg',- originalStorageKey: 'photo-1.jpg',- content: 'dolor sit amet',- originalSha256Hash: 'hash2',- },- ]);-- const { rows } = await db.run(sql`SELECT * FROM documents_fts;`);-- expect(rows).to.eql([- {- id: 'document-1',- name: 'Document 1',- content: 'lorem ipsum',- original_name: 'document-1.pdf',- },- {- id: 'document-2',- name: 'Photo 1',- content: 'dolor sit amet',- original_name: 'photo-1.jpg',- },- ]);-- const { rows: searchResults } = await db.run(sql`SELECT * FROM documents_fts WHERE documents_fts MATCH 'lorem';`);-- expect(searchResults).to.eql([- {- id: 'document-1',- name: 'Document 1',- content: 'lorem ipsum',- original_name: 'document-1.pdf',- },- ]);- });-- test('when updating a document, the corresponding row in the documents_fts table is updated', async () => {- const { db } = await createInMemoryDatabase({- users: [{ id: 'user-1', email: '[email protected]' }],- organizations: [{ id: 'organization-1', name: 'Organization 1' }],- organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],- });-- await db.insert(documentsTable).values([- {- id: 'document-1',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Document 1',- originalName: 'document-1.pdf',- originalStorageKey: 'document-1.pdf',- content: 'lorem ipsum',- originalSha256Hash: 'hash1',- },- {- id: 'document-2',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Photo 1',- originalName: 'photo-1.jpg',- originalStorageKey: 'photo-1.jpg',- content: 'dolor sit amet',- originalSha256Hash: 'hash2',- },- ]);-- await db.update(documentsTable).set({ content: 'foo bar baz' }).where(eq(documentsTable.id, 'document-1'));-- const { rows } = await db.run(sql`SELECT * FROM documents_fts;`);-- expect(rows).to.eql([- {- id: 'document-1',- name: 'Document 1',- content: 'foo bar baz',- original_name: 'document-1.pdf',- },- {- id: 'document-2',- name: 'Photo 1',- content: 'dolor sit amet',- original_name: 'photo-1.jpg',- },- ]);-- const { rows: searchResults } = await db.run(sql`SELECT * FROM documents_fts WHERE documents_fts MATCH 'foo';`);-- expect(searchResults).to.eql([- {- id: 'document-1',- name: 'Document 1',- content: 'foo bar baz',- original_name: 'document-1.pdf',- },- ]);- });-- test('when deleting a document, the corresponding row in the documents_fts table is deleted', async () => {- const { db } = await createInMemoryDatabase({- users: [{ id: 'user-1', email: '[email protected]' }],- organizations: [{ id: 'organization-1', name: 'Organization 1' }],- organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],- });-- await db.insert(documentsTable).values([- {- id: 'document-1',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Document 1',- originalName: 'document-1.pdf',- originalStorageKey: 'document-1.pdf',- content: 'lorem ipsum',- originalSha256Hash: 'hash1',- },- {- id: 'document-2',- organizationId: 'organization-1',- createdBy: 'user-1',- mimeType: 'application/pdf',- name: 'Photo 1',- originalName: 'photo-1.jpg',- originalStorageKey: 'photo-1.jpg',- content: 'dolor sit amet',- originalSha256Hash: 'hash2',- },- ]);-- await db.delete(documentsTable).where(eq(documentsTable.id, 'document-1'));-- const { rows } = await db.run(sql`SELECT * FROM documents_fts;`);-- expect(rows).to.eql([- {- id: 'document-2',- name: 'Photo 1',- content: 'dolor sit amet',- original_name: 'photo-1.jpg',- },- ]);-- const { rows: searchResults } = await db.run(sql`SELECT * FROM documents_fts WHERE documents_fts MATCH 'lorem';`);-- expect(searchResults).to.eql([]);- });- });- });-});diff --git a/apps/papra-server/src/modules/documents/documents.usecases.test.ts b/apps/papra-server/src/modules/documents/documents.usecases.test.tsindex be30976d..74d9b3c7 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.test.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.test.ts@@ -629,18 +629,10 @@ describe('documents usecases', () => {});expect(- eventServices.getEmittedEvents(),- ).to.eql([{- eventName: 'document.updated',- payload: {- changes: {- content: 'hello world',- },- documentId: 'document-1',- organizationId: 'organization-1',- userId: undefined,- },- }]);+ eventServices.getEmittedEvents().map(({ eventName }) => (eventName)),+ ).to.eql([+ 'document.updated',+ ]);});});@@ -821,6 +813,8 @@ describe('documents usecases', () => {originalName: 'file-1.txt',originalSha256Hash: 'hash',content: 'Original content',+ createdAt: new Date('2025-12-10'),+ updatedAt: new Date('2025-12-11'),}],});@@ -838,18 +832,40 @@ describe('documents usecases', () => {expect(eventServices.getEmittedEvents(),- ).to.eql([{- eventName: 'document.updated',- payload: {- changes: {- content: 'Updated content',- name: 'new-name.txt',+ ).to.eql(+ [+ {+ eventName: 'document.updated',+ payload: {+ changes: {+ content: 'Updated content',+ name: 'new-name.txt',+ },+ document: {+ content: 'Updated content',+ createdAt: new Date('2025-12-10'),+ createdBy: null,+ deletedAt: null,+ deletedBy: null,+ fileEncryptionAlgorithm: null,+ fileEncryptionKekVersion: null,+ fileEncryptionKeyWrapped: null,+ id: 'document-1',+ isDeleted: false,+ mimeType: 'text/plain',+ name: 'new-name.txt',+ organizationId: 'organization-1',+ originalName: 'file-1.txt',+ originalSha256Hash: 'hash',+ originalSize: 0,+ originalStorageKey: 'organization-1/originals/document-1.txt',+ updatedAt: new Date('2025-12-11'),+ },+ userId: 'user-1',+ },},- documentId: 'document-1',- organizationId: 'organization-1',- userId: 'user-1',- },- }]);+ ],+ );});});});diff --git a/apps/papra-server/src/modules/documents/documents.usecases.ts b/apps/papra-server/src/modules/documents/documents.usecases.tsindex b7c04922..2983807b 100644--- a/apps/papra-server/src/modules/documents/documents.usecases.ts+++ b/apps/papra-server/src/modules/documents/documents.usecases.ts@@ -324,6 +324,8 @@ async function createNewDocument({throw error;}+ const { document } = result;+await taskServices.scheduleJob({taskName: 'extract-document-file-content',data: { documentId, organizationId, ocrLanguages },@@ -331,7 +333,7 @@ async function createNewDocument({logger.info({ documentId, userId, organizationId, mimeType }, 'Document created');- return { document: result.document };+ return { document };}export async function getDocumentOrThrow({@@ -590,7 +592,7 @@ export async function updateDocument({eventServices.emitEvent({eventName: 'document.updated',- payload: { documentId, organizationId, userId, changes },+ payload: { userId, changes, document },});return { document };diff --git a/apps/papra-server/src/modules/documents/events/activity-log.document-updated.ts b/apps/papra-server/src/modules/documents/events/activity-log.document-updated.tsindex 4421844d..1568bb82 100644--- a/apps/papra-server/src/modules/documents/events/activity-log.document-updated.ts+++ b/apps/papra-server/src/modules/documents/events/activity-log.document-updated.ts@@ -15,9 +15,9 @@ export function registerInsertActivityLogOnDocumentUpdatedHandler({eventServices.onEvent({eventName: 'document.updated',handlerName: 'insert-activity-log',- async handler({ documentId, changes, userId }) {+ async handler({ document, changes, userId }) {await registerDocumentActivityLog({- documentId,+ documentId: document.id,event: 'updated',userId,documentActivityRepository,diff --git a/apps/papra-server/src/modules/documents/events/webhook.document-updated.ts b/apps/papra-server/src/modules/documents/events/webhook.document-updated.tsindex df329ad8..e12a8256 100644--- a/apps/papra-server/src/modules/documents/events/webhook.document-updated.ts+++ b/apps/papra-server/src/modules/documents/events/webhook.document-updated.ts@@ -15,14 +15,14 @@ export function registerTriggerWebhooksOnDocumentUpdatedHandler({eventServices.onEvent({eventName: 'document.updated',handlerName: 'trigger-webhooks',- async handler({ documentId, organizationId, changes }) {+ async handler({ document, changes }) {await triggerWebhooks({webhookRepository,- organizationId,+ organizationId: document.organizationId,event: 'document:updated',payload: {- documentId,- organizationId,+ documentId: document.id,+ organizationId: document.organizationId,...changes,},});diff --git a/apps/papra-server/src/start.ts b/apps/papra-server/src/start.tsindex a02004c5..08e1e381 100644--- a/apps/papra-server/src/start.ts+++ b/apps/papra-server/src/start.ts@@ -63,7 +63,7 @@ async function buildServices({ config }: { config: Config }): Promise<GlobalDepe// --- Services initializationawait taskServices.initialize();- registerEventHandlers({ eventServices, trackingServices, db });+ registerEventHandlers({ eventServices, trackingServices, db, documentSearchServices });return {config,