The minimalistic document archiving platform.
refactor(search): mutualize search function in dedicated fts5 repository (#658)
Corentin Thomasset334fcbdee479
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.tsindex fe5808d2..ac267c7e 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts@@ -1,15 +1,15 @@import type { Database } from '../../../app/database/database.types';import type { DocumentSearchServices } from '../document-search.types';-import { createDocumentsRepository } from '../../documents.repository';import { DATABASE_FTS5_DOCUMENT_SEARCH_PROVIDER_NAME } from './database-fts5.document-search-provider.constants';+import { createDocumentSearchRepository } from './database-fts5.repository';export function createDatabaseFts5DocumentSearchServices({ db }: { db: Database }): DocumentSearchServices {- const documentsRepository = createDocumentsRepository({ db });+ const documentsSearchRepository = createDocumentSearchRepository({ db });return {name: DATABASE_FTS5_DOCUMENT_SEARCH_PROVIDER_NAME,searchDocuments: async ({ searchQuery, organizationId, pageIndex, pageSize }) => {- const { documents } = await documentsRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });+ const { documents } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });return {searchResults: {diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.tsnew file mode 100644index 00000000..22c7e7ca--- /dev/null+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts@@ -0,0 +1,36 @@+import { describe, expect, test } from 'vitest';+import { createInMemoryDatabase } from '../../../app/database/database.test-utils';+import { ORGANIZATION_ROLES } from '../../../organizations/organizations.constants';+import { createDocumentSearchRepository } from './database-fts5.repository';++describe('database-fts5 repository', () => {+ describe('searchOrganizationDocuments', () => {+ test('provides full text search on document name, original name, and content', async () => {+ const { db } = await createInMemoryDatabase({+ users: [{ id: 'user-1', email: '[email protected]' }],+ organizations: [{ id: 'organization-1', name: 'Organization 1' }],+ organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],+ documents: [+ { id: 'doc-1', organizationId: 'organization-1', createdBy: 'user-1', name: 'Document 1', originalName: 'document-1.pdf', content: 'lorem ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1' },+ { id: 'doc-2', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 2', originalName: 'document-2.pdf', content: 'lorem', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash2' },+ { id: 'doc-3', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 3', originalName: 'document-3.pdf', content: 'ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash3' },+ ],+ });++ // Rebuild the FTS index since we are using an in-memory database+ await db.$client.execute(`INSERT INTO documents_fts(documents_fts) VALUES('rebuild');`);++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ const { documents } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'organization-1',+ searchQuery: 'lorem',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(documents).to.have.length(2);+ expect(documents.map(doc => doc.id)).to.eql(['doc-2', 'doc-1']);+ });+ });+});diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.tsnew file mode 100644index 00000000..2cecfd14--- /dev/null+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts@@ -0,0 +1,34 @@+import type { Database } from '../../../app/database/database.types';+import { injectArguments } from '@corentinth/chisels';+import { sql } from 'drizzle-orm';+import { documentsTable } from '../../documents.table';++export type DocumentSearchRepository = ReturnType<typeof createDocumentSearchRepository>;++export function createDocumentSearchRepository({ db }: { db: Database }) {+ return injectArguments({+ searchOrganizationDocuments,+ }, { db });+}++async function searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize, db }: { organizationId: string; searchQuery: string; pageIndex: number; pageSize: number; db: Database }) {+ // TODO: extract this logic to a tested function+ // when searchquery is a single word, we append a wildcard to it to make it a prefix search+ const cleanedSearchQuery = searchQuery.replace(/"/g, '').replace(/\*/g, '').trim();+ const formattedSearchQuery = cleanedSearchQuery.includes(' ') ? cleanedSearchQuery : `${cleanedSearchQuery}*`;++ const result = await db.run(sql`+ SELECT * FROM ${documentsTable}+ JOIN documents_fts ON documents_fts.id = ${documentsTable.id}+ WHERE ${documentsTable.organizationId} = ${organizationId}+ AND ${documentsTable.isDeleted} = 0+ AND documents_fts MATCH ${formattedSearchQuery}+ ORDER BY rank+ LIMIT ${pageSize}+ OFFSET ${pageIndex * pageSize}+ `);++ return {+ documents: result.rows as unknown as (typeof documentsTable.$inferSelect)[],+ };+}diff --git a/apps/papra-server/src/modules/documents/documents.repository.test.ts b/apps/papra-server/src/modules/documents/documents.repository.test.tsindex 126094b5..01a96e42 100644--- a/apps/papra-server/src/modules/documents/documents.repository.test.ts+++ b/apps/papra-server/src/modules/documents/documents.repository.test.ts@@ -1,4 +1,3 @@-import { map } from 'lodash-es';import { describe, expect, test } from 'vitest';import { createInMemoryDatabase } from '../app/database/database.test-utils';import { ORGANIZATION_ROLES } from '../organizations/organizations.constants';@@ -105,36 +104,6 @@ describe('documents repository', () => {});});- describe('searchOrganizationDocuments', () => {- test('provides full text search on document name, original name, and content', async () => {- const { db } = await createInMemoryDatabase({- users: [{ id: 'user-1', email: '[email protected]' }],- organizations: [{ id: 'organization-1', name: 'Organization 1' }],- organizationMembers: [{ organizationId: 'organization-1', userId: 'user-1', role: ORGANIZATION_ROLES.OWNER }],- documents: [- { id: 'doc-1', organizationId: 'organization-1', createdBy: 'user-1', name: 'Document 1', originalName: 'document-1.pdf', content: 'lorem ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1' },- { id: 'doc-2', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 2', originalName: 'document-2.pdf', content: 'lorem', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash2' },- { id: 'doc-3', organizationId: 'organization-1', createdBy: 'user-1', name: 'File 3', originalName: 'document-3.pdf', content: 'ipsum', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash3' },- ],- });-- // Rebuild the FTS index since we are using an in-memory database- await db.$client.execute(`INSERT INTO documents_fts(documents_fts) VALUES('rebuild');`);-- const documentsRepository = createDocumentsRepository({ db });-- const { documents } = await documentsRepository.searchOrganizationDocuments({- organizationId: 'organization-1',- searchQuery: 'lorem',- pageIndex: 0,- pageSize: 10,- });-- expect(documents).to.have.length(2);- expect(map(documents, 'id')).to.eql(['doc-2', 'doc-1']);- });- });-describe('getOrganizationStats', () => {test('retrieve document count and total size for an organization', async () => {const { db } = await createInMemoryDatabase({diff --git a/apps/papra-server/src/modules/documents/documents.repository.ts b/apps/papra-server/src/modules/documents/documents.repository.tsindex b9e03046..13f419da 100644--- a/apps/papra-server/src/modules/documents/documents.repository.ts+++ b/apps/papra-server/src/modules/documents/documents.repository.ts@@ -26,7 +26,6 @@ export function createDocumentsRepository({ db }: { db: Database }) {softDeleteDocument,getOrganizationDocumentsCount,getOrganizationDeletedDocumentsCount,- searchOrganizationDocuments,restoreDocument,hardDeleteDocument,getExpiredDeletedDocuments,@@ -307,28 +306,6 @@ async function getExpiredDeletedDocuments({ db, expirationDelayInDays, now = new};}-async function searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize, db }: { organizationId: string; searchQuery: string; pageIndex: number; pageSize: number; db: Database }) {- // TODO: extract this logic to a tested function- // when searchquery is a single word, we append a wildcard to it to make it a prefix search- const cleanedSearchQuery = searchQuery.replace(/"/g, '').replace(/\*/g, '').trim();- const formattedSearchQuery = cleanedSearchQuery.includes(' ') ? cleanedSearchQuery : `${cleanedSearchQuery}*`;-- const result = await db.run(sql`- SELECT * FROM ${documentsTable}- JOIN documents_fts ON documents_fts.id = ${documentsTable.id}- WHERE ${documentsTable.organizationId} = ${organizationId}- AND ${documentsTable.isDeleted} = 0- AND documents_fts MATCH ${formattedSearchQuery}- ORDER BY rank- LIMIT ${pageSize}- OFFSET ${pageIndex * pageSize}- `);-- return {- documents: result.rows as unknown as (typeof documentsTable.$inferSelect)[],- };-}-async function getOrganizationStats({ organizationId, db }: { organizationId: string; db: Database }) {const [record] = await db.select({