The minimalistic document archiving platform.
feat(search): add total count to search results (#756)
Corentin Thomassetf4102e5884c3
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.tsindex 7e4d8c4a..a4b534de 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts@@ -10,11 +10,12 @@ export function createDatabaseFts5DocumentSearchServices({ db }: { db: Databasename: DATABASE_FTS5_DOCUMENT_SEARCH_PROVIDER_NAME,searchDocuments: async ({ searchQuery, organizationId, pageIndex, pageSize }) => {- const { documents } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });+ const { documents, totalCount } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });return {searchResults: {- documents: documents.map(({ id, name }) => ({ id, name })),+ documents,+ totalCount,},};},diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.tsindex ba949bb0..506e1b3b 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts@@ -73,6 +73,106 @@ describe('database-fts5 repository', () => {expect(searchResults2.map(doc => doc.id)).to.eql(['doc_1']);});+ test('the total count of search results is returned for pagination', async () => {+ const documents = Array.from({ length: 15 }).map((_, index) => ({+ id: `doc_${index + 1}`,+ organizationId: 'org_1',+ name: `Document ${index + 1}`,+ originalName: `document-${index + 1}.pdf`,+ content: 'lorem ipsum dolor sit amet',+ originalStorageKey: '',+ mimeType: 'application/pdf',+ originalSha256Hash: `hash${index + 1}`,+ isDeleted: false,+ }));++ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'org_1', name: 'Organization 1' }],+ documents,+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ await Promise.all(+ documents.map(async document => documentsSearchRepository.indexDocument({ document })),+ );++ const { totalCount, documents: searchDocuments } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'org_1',+ searchQuery: 'lorem',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(totalCount).to.equal(15);+ expect(searchDocuments).to.have.length(10);+ });++ test('no results means total count is zero', async () => {+ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'org_1', name: 'Organization 1' }],+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ const { totalCount, documents: searchDocuments } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'org_1',+ searchQuery: 'nonexistent',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(totalCount).to.equal(0);+ expect(searchDocuments).to.have.length(0);+ });++ test('pagination permits retrieving subsequent pages of results', async () => {+ const documents = Array.from({ length: 25 }).map((_, index) => ({+ id: `doc_${index + 1}`,+ organizationId: 'org_1',+ name: `Document ${index + 1}`,+ originalName: `document-${index + 1}.pdf`,+ content: 'lorem ipsum dolor sit amet',+ originalStorageKey: '',+ mimeType: 'application/pdf',+ originalSha256Hash: `hash${index + 1}`,+ isDeleted: false,+ }));++ const { db } = await createInMemoryDatabase({+ organizations: [{ id: 'org_1', name: 'Organization 1' }],+ documents,+ });++ const documentsSearchRepository = createDocumentSearchRepository({ db });++ await Promise.all(+ documents.map(async document => documentsSearchRepository.indexDocument({ document })),+ );++ const { documents: firstPageDocuments, totalCount: firstTotalCount } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'org_1',+ searchQuery: 'lorem',+ pageIndex: 0,+ pageSize: 10,+ });++ expect(firstPageDocuments).to.have.length(10);+ expect(firstTotalCount).to.equal(25);+ expect(firstPageDocuments.map(doc => doc.id)).to.eql(['doc_1', 'doc_2', 'doc_3', 'doc_4', 'doc_5', 'doc_6', 'doc_7', 'doc_8', 'doc_9', 'doc_10']);++ const { documents: secondPageDocuments, totalCount: secondTotalCount } = await documentsSearchRepository.searchOrganizationDocuments({+ organizationId: 'org_1',+ searchQuery: 'lorem',+ pageIndex: 1,+ pageSize: 5,+ });++ expect(secondPageDocuments).to.have.length(5);+ expect(secondTotalCount).to.equal(25);+ expect(secondPageDocuments.map(doc => doc.id)).to.eql(['doc_6', 'doc_7', 'doc_8', 'doc_9', 'doc_10']);+ });+describe('search query can have filters', async () => {const documents = [{ id: 'doc_1', organizationId: 'org_1', name: 'Invoice cloudflare', originalName: 'document-1.pdf', content: 'Cloudflare invoice, hosting 5€', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1', isDeleted: false },@@ -183,7 +283,6 @@ describe('database-fts5 repository', () => {// Should only return documents from org_1, not org_2expect(org1Results).to.have.length(1);expect(org1Results.map(doc => doc.id)).to.eql(['doc_1']);- expect(org1Results.every(doc => doc.organizationId === 'org_1')).to.eql(true);// Search in org_2 with the same termconst { documents: org2Results } = await documentsSearchRepository.searchOrganizationDocuments({@@ -196,7 +295,6 @@ describe('database-fts5 repository', () => {// Should only return documents from org_2, not org_1expect(org2Results).to.have.length(2);expect(org2Results.map(doc => doc.id).toSorted()).to.eql(['doc_3', 'doc_4']);- expect(org2Results.every(doc => doc.organizationId === 'org_2')).to.eql(true);// Search for organization-specific content should not leak across organizationsconst { documents: org1SpecificSearch } = await documentsSearchRepository.searchOrganizationDocuments({diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.tsindex 60f3013d..5cdd7fa7 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts@@ -1,7 +1,7 @@import type { Database } from '../../../app/database/database.types';import type { DocumentSearchableData } from '../document-search.types';import { injectArguments } from '@corentinth/chisels';-import { asc, eq } from 'drizzle-orm';+import { asc, eq, sql } from 'drizzle-orm';import { omitUndefined } from '../../../shared/utils';import { documentsTable } from '../../documents.table';import { makeSearchWhereClause } from './database-fts5.repository.models';@@ -21,10 +21,11 @@ export function createDocumentSearchRepository({ db }: { db: Database }) {async function searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize, db }: { organizationId: string; searchQuery: string; pageIndex: number; pageSize: number; db: Database }) {const { searchWhereClause } = makeSearchWhereClause({ organizationId, query: searchQuery, db });- const query = db.selectDistinct({+ const documentsWithTotalCount = await db.selectDistinct({id: documentsTable.id,organizationId: documentsTable.organizationId,name: documentsTable.name,+ totalCount: sql<number>`COUNT(*) OVER()`,}).from(documentsTable).where(searchWhereClause)@@ -32,11 +33,10 @@ async function searchOrganizationDocuments({ organizationId, searchQuery, pageIn.limit(pageSize).offset(pageIndex * pageSize);- // console.log(query.toSQL());+ const documents = documentsWithTotalCount.map(({ id, name }) => ({ id, name }));+ const totalCount = documentsWithTotalCount[0]?.totalCount ?? 0;- const documents = await query;-- return { documents };+ return { documents, totalCount };}async function indexDocument({ document, db }: { document: DocumentSearchableData; db: Database }) {diff --git a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts b/apps/papra-server/src/modules/documents/document-search/document-search.types.tsindex d007ade9..692dcf87 100644--- a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts+++ b/apps/papra-server/src/modules/documents/document-search/document-search.types.ts@@ -14,7 +14,7 @@ export type DocumentSearchServices = {organizationId: string;pageIndex: number;pageSize: number;- }) => Promise<{ searchResults: { documents: { id: string; name: string }[] } }>;+ }) => Promise<{ searchResults: { documents: { id: string; name: string }[]; totalCount: number } }>;indexDocument: (args: { document: DocumentSearchableData }) => Promise<void>;updateDocument: (args: { documentId: string; document: Partial<Omit<DocumentSearchableData, 'id'>> }) => Promise<void>;diff --git a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.tsindex 6468fa3d..ee00291d 100644--- a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts+++ b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts@@ -18,7 +18,7 @@ function createTestSearchServices() {name: 'test-search-service',searchDocuments: async (args) => {methodsArgs.searchDocuments.push(args);- return { searchResults: { documents: [] } };+ return { searchResults: { documents: [], totalCount: 0 } };},indexDocument: async (args) => {methodsArgs.indexDocument.push(args);