The minimalistic document archiving platform.
feat(search): index document and organization ids in fts search index (#733)
Corentin Thomasset848671b98237
diff --git a/.changeset/eager-grapes-raise.md b/.changeset/eager-grapes-raise.mdnew file mode 100644index 00000000..ef03cc17--- /dev/null+++ b/.changeset/eager-grapes-raise.md@@ -0,0 +1,5 @@+---+"@papra/docker": patch+---++Improved search speed by using document and organization ids in indexdiff --git a/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts b/apps/papra-server/src/migrations/list/0002-documents-fts.migration.tsindex 8f51e127..f75ce62e 100644--- a/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts+++ b/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts@@ -6,6 +6,7 @@ export const documentsFtsMigration = {up: async ({ db }) => {await db.batch([+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4')`),db.run(sql`INSERT INTO documents_fts(id, name, original_name, content) SELECT id, name, original_name, content FROM documents`),db.run(sql`diff --git a/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.ts b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.tsnew file mode 100644index 00000000..2386a280--- /dev/null+++ b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.ts@@ -0,0 +1,39 @@+import { sql } from 'drizzle-orm';+import { describe, expect, test } from 'vitest';+import { setupDatabase } from '../../modules/app/database/database';+import { initialSchemaSetupMigration } from './0001-initial-schema-setup.migration';+import { documentsFtsMigration } from './0002-documents-fts.migration';+import { dropFts5TriggersMigration } from './0013-drop-fts-5-triggers.migration';+import { indexDocumentsFtsIdsMigration } from './0015-index-documents-fts-ids.migration';++describe('0015-index-documents-fts-ids migration', () => {+ describe('index-documents-fts-ids', () => {+ test('the recreation of the FTS table with document_id and organization_id included as searchable columns still permits full text search', async () => {+ const { db } = setupDatabase({ url: ':memory:' });+ // Recreate a documents table and populate it with sample data+ await initialSchemaSetupMigration.up({ db });++ await db.batch([+ db.run(sql`INSERT INTO organizations(id, name, created_at, updated_at) VALUES ('org_1', 'Test Organization', 0, 0)`),+ db.run(sql`INSERT INTO documents(id, organization_id, original_name, name, mime_type, original_storage_key, original_sha256_hash, content, created_at, updated_at)+ VALUES+ ('doc1', 'org_1', 'Test Document', 'Test Document', 'text/plain', 'key1', 'hash1', 'This is a sample document content about testing.',0,0),+ ('doc2', 'org_1', 'Another Document', 'Another Document', 'text/plain', 'key2', 'hash2', 'This document discusses database migrations.',0,0)+ `),+ ]);++ await documentsFtsMigration.up({ db });+ await dropFts5TriggersMigration.up({ db });++ const { rows: searchBeforeMigration } = await db.run(sql`SELECT id FROM documents_fts WHERE documents_fts = 'Test*'`);++ expect(searchBeforeMigration).to.eql([{ id: 'doc1' }]);++ await indexDocumentsFtsIdsMigration.up({ db });++ const { rows: searchAfterMigration } = await db.run(sql`SELECT document_id FROM documents_fts WHERE documents_fts = 'Test*'`);++ expect(searchAfterMigration).to.eql([{ document_id: 'doc1' }]);+ });+ });+});diff --git a/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.ts b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.tsnew file mode 100644index 00000000..840bc9a2--- /dev/null+++ b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.ts@@ -0,0 +1,38 @@+import type { Migration } from '../migrations.types';+import { sql } from 'drizzle-orm';++export const indexDocumentsFtsIdsMigration = {+ name: 'index-documents-fts-ids',++ up: async ({ db }) => {+ await db.batch([+ // Drop old FTS table+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),++ // Recreate FTS table with document_id and organization_id as indexed for efficient joins and removed original_name+ db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(+ document_id,+ organization_id,+ name,+ content,+ prefix='2 3 4'+ )`),++ // Populate the new FTS table+ db.run(sql`INSERT INTO documents_fts(document_id, organization_id, name, content)+ SELECT id, organization_id, name, content FROM documents`),+ ]);+ },++ down: async ({ db }) => {+ await db.batch([+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),++ // Recreate old FTS table with original schema+ db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4')`),++ // Populate with old structure+ db.run(sql`INSERT INTO documents_fts(id, name, original_name, content) SELECT id, name, original_name, content FROM documents`),+ ]);+ },+} satisfies Migration;diff --git a/apps/papra-server/src/migrations/migrations.registry.test.ts b/apps/papra-server/src/migrations/migrations.registry.test.tsindex 40b0b741..b320aa48 100644--- a/apps/papra-server/src/migrations/migrations.registry.test.ts+++ b/apps/papra-server/src/migrations/migrations.registry.test.ts@@ -100,7 +100,7 @@ describe('migrations registry', () => {CREATE TABLE "auth_verifications" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "identifier" text NOT NULL, "value" text NOT NULL, "expires_at" integer NOT NULL );CREATE TABLE "document_activity_log" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "document_id" text NOT NULL, "event" text NOT NULL, "event_data" text, "user_id" text, "tag_id" text, FOREIGN KEY ("document_id") REFERENCES "documents"("id") ON UPDATE cascade ON DELETE cascade, FOREIGN KEY ("user_id") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("tag_id") REFERENCES "tags"("id") ON UPDATE cascade ON DELETE set null );CREATE TABLE "documents" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "is_deleted" integer DEFAULT false NOT NULL, "deleted_at" integer, "organization_id" text NOT NULL, "created_by" text, "deleted_by" text, "original_name" text NOT NULL, "original_size" integer DEFAULT 0 NOT NULL, "original_storage_key" text NOT NULL, "original_sha256_hash" text NOT NULL, "name" text NOT NULL, "mime_type" text NOT NULL, "content" text DEFAULT '' NOT NULL, file_encryption_key_wrapped TEXT, file_encryption_kek_version TEXT, file_encryption_algorithm TEXT, FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON UPDATE cascade ON DELETE cascade, FOREIGN KEY ("created_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("deleted_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null );- CREATE VIRTUAL TABLE documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4');+ CREATE VIRTUAL TABLE documents_fts USING fts5( document_id, organization_id, name, content, prefix='2 3 4' );CREATE TABLE 'documents_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID;CREATE TABLE 'documents_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3);CREATE TABLE 'documents_fts_data'(id INTEGER PRIMARY KEY, block BLOB);diff --git a/apps/papra-server/src/migrations/migrations.registry.ts b/apps/papra-server/src/migrations/migrations.registry.tsindex 4d670ce5..342d9c31 100644--- a/apps/papra-server/src/migrations/migrations.registry.ts+++ b/apps/papra-server/src/migrations/migrations.registry.ts@@ -14,6 +14,7 @@ import { softDeleteOrganizationsMigration } from './list/0011-soft-delete-organiimport { taggingRuleConditionMatchModeMigration } from './list/0012-tagging-rule-condition-match-mode.migration';import { dropFts5TriggersMigration } from './list/0013-drop-fts-5-triggers.migration';import { twoFactorAuthenticationMigration } from './list/0014-two-factor-authentication.migration';+import { indexDocumentsFtsIdsMigration } from './list/0015-index-documents-fts-ids.migration';export const migrations: Migration[] = [initialSchemaSetupMigration,@@ -30,4 +31,5 @@ export const migrations: Migration[] = [taggingRuleConditionMatchModeMigration,dropFts5TriggersMigration,twoFactorAuthenticationMigration,+ indexDocumentsFtsIdsMigration,];diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.tsindex e4a47812..5cda28fc 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts@@ -27,10 +27,10 @@ async function searchOrganizationDocuments({ organizationId, searchQuery, pageIn.from(documentsTable).innerJoin(documentsFtsTable,- eq(documentsFtsTable.id, documentsTable.id),+ eq(documentsFtsTable.documentId, documentsTable.id),).where(and(- eq(documentsTable.organizationId, organizationId),+ eq(documentsFtsTable.organizationId, organizationId),eq(documentsTable.isDeleted, false),eq(documentsFtsTable, formattedSearchQuery), // Match and eq works the same for FTS5 virtual tables))@@ -45,9 +45,9 @@ async function indexDocument({ document, db }: { document: DocumentSearchableDatawait db.insert(documentsFtsTable).values({- id: document.id,+ documentId: document.id,+ organizationId: document.organizationId,name: document.name,- originalName: document.originalName,content: document.content,});}@@ -66,11 +66,11 @@ async function updateDocument({ documentId, document, db }: { documentId: stringawait db.update(documentsFtsTable).set(dataToUpdate)- .where(eq(documentsFtsTable.id, documentId));+ .where(eq(documentsFtsTable.documentId, documentId));}async function deleteDocument({ documentId, db }: { documentId: string; db: Database }) {await db.delete(documentsFtsTable)- .where(eq(documentsFtsTable.id, documentId));+ .where(eq(documentsFtsTable.documentId, documentId));}diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.tsindex c12dcf0f..b560ae26 100644--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts@@ -1,8 +1,8 @@import { sqliteTable, text } from 'drizzle-orm/sqlite-core';export const documentsFtsTable = sqliteTable('documents_fts', {- id: text('id').notNull(),+ documentId: text('document_id').notNull(),+ organizationId: text('organization_id').notNull(),name: text('name').notNull(),- originalName: text('original_name').notNull(),content: text('content').notNull(),});