Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(search): add total count to search results (#756)

Corentin Thomassetf4102e5884c3
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts
index 7e4d8c4a..a4b534de 100644
--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts
+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.document-search-provider.ts
@@ -10,11 +10,12 @@ export function createDatabaseFts5DocumentSearchServices({ db }: { db: Database
name: DATABASE_FTS5_DOCUMENT_SEARCH_PROVIDER_NAME,
searchDocuments: async ({ searchQuery, organizationId, pageIndex, pageSize }) => {
- const { documents } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });
+ const { documents, totalCount } = await documentsSearchRepository.searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize });
return {
searchResults: {
- documents: documents.map(({ id, name }) => ({ id, name })),
+ documents,
+ totalCount,
},
};
},
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts
index ba949bb0..506e1b3b 100644
--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts
+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.test.ts
@@ -73,6 +73,106 @@ describe('database-fts5 repository', () => {
expect(searchResults2.map(doc => doc.id)).to.eql(['doc_1']);
});
+ test('the total count of search results is returned for pagination', async () => {
+ const documents = Array.from({ length: 15 }).map((_, index) => ({
+ id: `doc_${index + 1}`,
+ organizationId: 'org_1',
+ name: `Document ${index + 1}`,
+ originalName: `document-${index + 1}.pdf`,
+ content: 'lorem ipsum dolor sit amet',
+ originalStorageKey: '',
+ mimeType: 'application/pdf',
+ originalSha256Hash: `hash${index + 1}`,
+ isDeleted: false,
+ }));
+
+ const { db } = await createInMemoryDatabase({
+ organizations: [{ id: 'org_1', name: 'Organization 1' }],
+ documents,
+ });
+
+ const documentsSearchRepository = createDocumentSearchRepository({ db });
+
+ await Promise.all(
+ documents.map(async document => documentsSearchRepository.indexDocument({ document })),
+ );
+
+ const { totalCount, documents: searchDocuments } = await documentsSearchRepository.searchOrganizationDocuments({
+ organizationId: 'org_1',
+ searchQuery: 'lorem',
+ pageIndex: 0,
+ pageSize: 10,
+ });
+
+ expect(totalCount).to.equal(15);
+ expect(searchDocuments).to.have.length(10);
+ });
+
+ test('no results means total count is zero', async () => {
+ const { db } = await createInMemoryDatabase({
+ organizations: [{ id: 'org_1', name: 'Organization 1' }],
+ });
+
+ const documentsSearchRepository = createDocumentSearchRepository({ db });
+
+ const { totalCount, documents: searchDocuments } = await documentsSearchRepository.searchOrganizationDocuments({
+ organizationId: 'org_1',
+ searchQuery: 'nonexistent',
+ pageIndex: 0,
+ pageSize: 10,
+ });
+
+ expect(totalCount).to.equal(0);
+ expect(searchDocuments).to.have.length(0);
+ });
+
+ test('pagination permits retrieving subsequent pages of results', async () => {
+ const documents = Array.from({ length: 25 }).map((_, index) => ({
+ id: `doc_${index + 1}`,
+ organizationId: 'org_1',
+ name: `Document ${index + 1}`,
+ originalName: `document-${index + 1}.pdf`,
+ content: 'lorem ipsum dolor sit amet',
+ originalStorageKey: '',
+ mimeType: 'application/pdf',
+ originalSha256Hash: `hash${index + 1}`,
+ isDeleted: false,
+ }));
+
+ const { db } = await createInMemoryDatabase({
+ organizations: [{ id: 'org_1', name: 'Organization 1' }],
+ documents,
+ });
+
+ const documentsSearchRepository = createDocumentSearchRepository({ db });
+
+ await Promise.all(
+ documents.map(async document => documentsSearchRepository.indexDocument({ document })),
+ );
+
+ const { documents: firstPageDocuments, totalCount: firstTotalCount } = await documentsSearchRepository.searchOrganizationDocuments({
+ organizationId: 'org_1',
+ searchQuery: 'lorem',
+ pageIndex: 0,
+ pageSize: 10,
+ });
+
+ expect(firstPageDocuments).to.have.length(10);
+ expect(firstTotalCount).to.equal(25);
+ expect(firstPageDocuments.map(doc => doc.id)).to.eql(['doc_1', 'doc_2', 'doc_3', 'doc_4', 'doc_5', 'doc_6', 'doc_7', 'doc_8', 'doc_9', 'doc_10']);
+
+ const { documents: secondPageDocuments, totalCount: secondTotalCount } = await documentsSearchRepository.searchOrganizationDocuments({
+ organizationId: 'org_1',
+ searchQuery: 'lorem',
+ pageIndex: 1,
+ pageSize: 5,
+ });
+
+ expect(secondPageDocuments).to.have.length(5);
+ expect(secondTotalCount).to.equal(25);
+ expect(secondPageDocuments.map(doc => doc.id)).to.eql(['doc_6', 'doc_7', 'doc_8', 'doc_9', 'doc_10']);
+ });
+
describe('search query can have filters', async () => {
const documents = [
{ id: 'doc_1', organizationId: 'org_1', name: 'Invoice cloudflare', originalName: 'document-1.pdf', content: 'Cloudflare invoice, hosting 5€', originalStorageKey: '', mimeType: 'application/pdf', originalSha256Hash: 'hash1', isDeleted: false },
@@ -183,7 +283,6 @@ describe('database-fts5 repository', () => {
// Should only return documents from org_1, not org_2
expect(org1Results).to.have.length(1);
expect(org1Results.map(doc => doc.id)).to.eql(['doc_1']);
- expect(org1Results.every(doc => doc.organizationId === 'org_1')).to.eql(true);
// Search in org_2 with the same term
const { documents: org2Results } = await documentsSearchRepository.searchOrganizationDocuments({
@@ -196,7 +295,6 @@ describe('database-fts5 repository', () => {
// Should only return documents from org_2, not org_1
expect(org2Results).to.have.length(2);
expect(org2Results.map(doc => doc.id).toSorted()).to.eql(['doc_3', 'doc_4']);
- expect(org2Results.every(doc => doc.organizationId === 'org_2')).to.eql(true);
// Search for organization-specific content should not leak across organizations
const { documents: org1SpecificSearch } = await documentsSearchRepository.searchOrganizationDocuments({
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
index 60f3013d..5cdd7fa7 100644
--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
@@ -1,7 +1,7 @@
import type { Database } from '../../../app/database/database.types';
import type { DocumentSearchableData } from '../document-search.types';
import { injectArguments } from '@corentinth/chisels';
-import { asc, eq } from 'drizzle-orm';
+import { asc, eq, sql } from 'drizzle-orm';
import { omitUndefined } from '../../../shared/utils';
import { documentsTable } from '../../documents.table';
import { makeSearchWhereClause } from './database-fts5.repository.models';
@@ -21,10 +21,11 @@ export function createDocumentSearchRepository({ db }: { db: Database }) {
async function searchOrganizationDocuments({ organizationId, searchQuery, pageIndex, pageSize, db }: { organizationId: string; searchQuery: string; pageIndex: number; pageSize: number; db: Database }) {
const { searchWhereClause } = makeSearchWhereClause({ organizationId, query: searchQuery, db });
- const query = db.selectDistinct({
+ const documentsWithTotalCount = await db.selectDistinct({
id: documentsTable.id,
organizationId: documentsTable.organizationId,
name: documentsTable.name,
+ totalCount: sql<number>`COUNT(*) OVER()`,
})
.from(documentsTable)
.where(searchWhereClause)
@@ -32,11 +33,10 @@ async function searchOrganizationDocuments({ organizationId, searchQuery, pageIn
.limit(pageSize)
.offset(pageIndex * pageSize);
- // console.log(query.toSQL());
+ const documents = documentsWithTotalCount.map(({ id, name }) => ({ id, name }));
+ const totalCount = documentsWithTotalCount[0]?.totalCount ?? 0;
- const documents = await query;
-
- return { documents };
+ return { documents, totalCount };
}
async function indexDocument({ document, db }: { document: DocumentSearchableData; db: Database }) {
diff --git a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts b/apps/papra-server/src/modules/documents/document-search/document-search.types.ts
index d007ade9..692dcf87 100644
--- a/apps/papra-server/src/modules/documents/document-search/document-search.types.ts
+++ b/apps/papra-server/src/modules/documents/document-search/document-search.types.ts
@@ -14,7 +14,7 @@ export type DocumentSearchServices = {
organizationId: string;
pageIndex: number;
pageSize: number;
- }) => Promise<{ searchResults: { documents: { id: string; name: string }[] } }>;
+ }) => Promise<{ searchResults: { documents: { id: string; name: string }[]; totalCount: number } }>;
indexDocument: (args: { document: DocumentSearchableData }) => Promise<void>;
updateDocument: (args: { documentId: string; document: Partial<Omit<DocumentSearchableData, 'id'>> }) => Promise<void>;
diff --git a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts
index 6468fa3d..ee00291d 100644
--- a/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts
+++ b/apps/papra-server/src/modules/documents/document-search/events/sync-document-search.handlers.test.ts
@@ -18,7 +18,7 @@ function createTestSearchServices() {
name: 'test-search-service',
searchDocuments: async (args) => {
methodsArgs.searchDocuments.push(args);
- return { searchResults: { documents: [] } };
+ return { searchResults: { documents: [], totalCount: 0 } };
},
indexDocument: async (args) => {
methodsArgs.indexDocument.push(args);